2024-11-11 20:39:31,077 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 20:39:31,094 main DEBUG Took 0.014335 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 20:39:31,095 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 20:39:31,096 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 20:39:31,097 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 20:39:31,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,111 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 20:39:31,135 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,137 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,138 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,140 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,142 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,143 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,145 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,147 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,149 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,153 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,154 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,156 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,158 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,159 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:39:31,162 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,163 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 20:39:31,165 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:39:31,167 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 20:39:31,170 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 20:39:31,171 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 20:39:31,173 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 20:39:31,174 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 20:39:31,186 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 20:39:31,194 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 20:39:31,198 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 20:39:31,199 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 20:39:31,199 main DEBUG createAppenders(={Console}) 2024-11-11 20:39:31,200 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 20:39:31,201 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 20:39:31,201 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 20:39:31,202 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 20:39:31,203 main DEBUG OutputStream closed 2024-11-11 20:39:31,204 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 20:39:31,204 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 20:39:31,204 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 20:39:31,313 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 20:39:31,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 20:39:31,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 20:39:31,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 20:39:31,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 20:39:31,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 20:39:31,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 20:39:31,323 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 20:39:31,323 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 20:39:31,324 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 20:39:31,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 20:39:31,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 20:39:31,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 20:39:31,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 20:39:31,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 20:39:31,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 20:39:31,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 20:39:31,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 20:39:31,340 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 20:39:31,341 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 20:39:31,341 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 20:39:31,343 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T20:39:31,739 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2 2024-11-11 20:39:31,745 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 20:39:31,746 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T20:39:31,758 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-11T20:39:31,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T20:39:31,825 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a, deleteOnExit=true 2024-11-11T20:39:31,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T20:39:31,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/test.cache.data in system properties and HBase conf 2024-11-11T20:39:31,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T20:39:31,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/hadoop.log.dir in system properties and HBase conf 2024-11-11T20:39:31,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T20:39:31,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T20:39:31,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T20:39:31,981 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T20:39:32,137 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:39:32,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T20:39:32,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T20:39:32,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T20:39:32,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T20:39:32,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T20:39:32,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T20:39:32,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T20:39:32,149 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T20:39:32,149 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T20:39:32,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/nfs.dump.dir in system properties and HBase conf 2024-11-11T20:39:32,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/java.io.tmpdir in system properties and HBase conf 2024-11-11T20:39:32,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T20:39:32,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T20:39:32,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T20:39:33,117 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T20:39:33,216 INFO [Time-limited test {}] log.Log(170): Logging initialized @3095ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T20:39:33,318 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:39:33,413 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T20:39:33,451 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T20:39:33,452 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T20:39:33,454 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T20:39:33,469 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:39:33,476 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/hadoop.log.dir/,AVAILABLE} 2024-11-11T20:39:33,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T20:39:33,698 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/java.io.tmpdir/jetty-localhost-38065-hadoop-hdfs-3_4_1-tests_jar-_-any-1688128760101330532/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T20:39:33,706 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:38065} 2024-11-11T20:39:33,706 INFO [Time-limited test {}] server.Server(415): Started @3586ms 2024-11-11T20:39:34,096 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:39:34,104 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T20:39:34,105 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T20:39:34,106 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T20:39:34,106 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T20:39:34,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/hadoop.log.dir/,AVAILABLE} 2024-11-11T20:39:34,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T20:39:34,224 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/java.io.tmpdir/jetty-localhost-34825-hadoop-hdfs-3_4_1-tests_jar-_-any-1772467338737855264/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:39:34,225 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34825} 2024-11-11T20:39:34,225 INFO [Time-limited test {}] server.Server(415): Started @4105ms 2024-11-11T20:39:34,284 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T20:39:34,807 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a/data/data1/current/BP-178686629-172.17.0.2-1731357572864/current, will proceed with Du for space computation calculation, 2024-11-11T20:39:34,809 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a/data/data2/current/BP-178686629-172.17.0.2-1731357572864/current, will proceed with Du for space computation calculation, 2024-11-11T20:39:34,895 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T20:39:34,994 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d9b1b9f5f31d961 with lease ID 0x25dd6b8cee61657b: Processing first storage report for DS-744ed6d9-ac4c-4242-a538-2c1cc65f0abb from datanode DatanodeRegistration(127.0.0.1:38937, datanodeUuid=99528e4a-9935-46c8-bffc-ee2a5a7e68a2, infoPort=44507, infoSecurePort=0, ipcPort=40417, storageInfo=lv=-57;cid=testClusterID;nsid=1766808156;c=1731357572864) 2024-11-11T20:39:34,995 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d9b1b9f5f31d961 with lease ID 0x25dd6b8cee61657b: from storage DS-744ed6d9-ac4c-4242-a538-2c1cc65f0abb node DatanodeRegistration(127.0.0.1:38937, datanodeUuid=99528e4a-9935-46c8-bffc-ee2a5a7e68a2, infoPort=44507, infoSecurePort=0, ipcPort=40417, storageInfo=lv=-57;cid=testClusterID;nsid=1766808156;c=1731357572864), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-11T20:39:34,996 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d9b1b9f5f31d961 with lease ID 0x25dd6b8cee61657b: Processing first storage report for DS-2e098718-7dd7-4d41-97f7-404a9188785a from datanode DatanodeRegistration(127.0.0.1:38937, datanodeUuid=99528e4a-9935-46c8-bffc-ee2a5a7e68a2, infoPort=44507, infoSecurePort=0, ipcPort=40417, storageInfo=lv=-57;cid=testClusterID;nsid=1766808156;c=1731357572864) 2024-11-11T20:39:34,996 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d9b1b9f5f31d961 with lease ID 0x25dd6b8cee61657b: from storage DS-2e098718-7dd7-4d41-97f7-404a9188785a node DatanodeRegistration(127.0.0.1:38937, datanodeUuid=99528e4a-9935-46c8-bffc-ee2a5a7e68a2, infoPort=44507, infoSecurePort=0, ipcPort=40417, storageInfo=lv=-57;cid=testClusterID;nsid=1766808156;c=1731357572864), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T20:39:35,055 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2 2024-11-11T20:39:35,154 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a/zookeeper_0, clientPort=60247, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T20:39:35,167 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60247 2024-11-11T20:39:35,184 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:39:35,187 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:39:35,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741825_1001 (size=7) 2024-11-11T20:39:35,875 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30 with version=8 2024-11-11T20:39:35,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/hbase-staging 2024-11-11T20:39:35,986 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T20:39:36,235 INFO [Time-limited test {}] client.ConnectionUtils(128): master/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:39:36,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:39:36,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:39:36,256 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:39:36,257 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:39:36,257 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:39:36,401 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T20:39:36,455 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T20:39:36,463 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T20:39:36,467 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:39:36,492 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 995 (auto-detected) 2024-11-11T20:39:36,493 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T20:39:36,512 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35885 2024-11-11T20:39:36,533 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35885 connecting to ZooKeeper ensemble=127.0.0.1:60247 2024-11-11T20:39:36,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358850x0, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:39:36,563 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35885-0x1003083643e0000 connected 2024-11-11T20:39:36,587 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:39:36,589 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:39:36,598 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:39:36,602 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30, hbase.cluster.distributed=false 2024-11-11T20:39:36,624 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:39:36,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35885 2024-11-11T20:39:36,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35885 2024-11-11T20:39:36,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35885 2024-11-11T20:39:36,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35885 2024-11-11T20:39:36,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35885 2024-11-11T20:39:36,738 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:39:36,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:39:36,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:39:36,740 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:39:36,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:39:36,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:39:36,743 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:39:36,746 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:39:36,747 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44271 2024-11-11T20:39:36,748 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44271 connecting to ZooKeeper ensemble=127.0.0.1:60247 2024-11-11T20:39:36,750 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:39:36,752 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:39:36,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442710x0, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:39:36,758 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442710x0, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:39:36,758 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44271-0x1003083643e0001 connected 2024-11-11T20:39:36,762 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:39:36,771 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:39:36,774 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T20:39:36,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:39:36,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44271 2024-11-11T20:39:36,785 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44271 2024-11-11T20:39:36,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44271 2024-11-11T20:39:36,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44271 2024-11-11T20:39:36,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44271 2024-11-11T20:39:36,805 DEBUG [M:0;51ca66f7ee3c:35885 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;51ca66f7ee3c:35885 2024-11-11T20:39:36,806 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:36,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:39:36,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:39:36,814 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:36,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:36,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T20:39:36,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:36,837 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T20:39:36,839 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/51ca66f7ee3c,35885,1731357576039 from backup master directory 2024-11-11T20:39:36,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:39:36,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:36,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:39:36,842 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:39:36,843 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:36,845 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T20:39:36,846 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T20:39:36,900 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/hbase.id] with ID: 7eb98061-214b-40f9-8a48-d23c5a1d1708 2024-11-11T20:39:36,900 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/.tmp/hbase.id 2024-11-11T20:39:36,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741826_1002 (size=42) 2024-11-11T20:39:37,313 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/.tmp/hbase.id]:[hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/hbase.id] 2024-11-11T20:39:37,368 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:39:37,374 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T20:39:37,400 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-11T20:39:37,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:37,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:37,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741827_1003 (size=196) 2024-11-11T20:39:37,840 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:39:37,843 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T20:39:37,860 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:37,863 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:39:37,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741828_1004 (size=1189) 2024-11-11T20:39:38,314 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store 2024-11-11T20:39:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741829_1005 (size=34) 2024-11-11T20:39:38,742 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T20:39:38,746 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:39:38,747 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:39:38,748 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:39:38,748 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:39:38,750 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:39:38,750 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:39:38,750 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:39:38,752 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731357578747Disabling compacts and flushes for region at 1731357578747Disabling writes for close at 1731357578750 (+3 ms)Writing region close event to WAL at 1731357578750Closed at 1731357578750 2024-11-11T20:39:38,754 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/.initializing 2024-11-11T20:39:38,755 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/WALs/51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:38,764 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:39:38,778 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C35885%2C1731357576039, suffix=, logDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/WALs/51ca66f7ee3c,35885,1731357576039, archiveDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/oldWALs, maxLogs=10 2024-11-11T20:39:38,799 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/WALs/51ca66f7ee3c,35885,1731357576039/51ca66f7ee3c%2C35885%2C1731357576039.1731357578782, exclude list is [], retry=0 2024-11-11T20:39:38,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38937,DS-744ed6d9-ac4c-4242-a538-2c1cc65f0abb,DISK] 2024-11-11T20:39:38,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T20:39:38,848 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/WALs/51ca66f7ee3c,35885,1731357576039/51ca66f7ee3c%2C35885%2C1731357576039.1731357578782 2024-11-11T20:39:38,849 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507)] 2024-11-11T20:39:38,850 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:39:38,850 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:39:38,854 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,855 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,910 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T20:39:38,913 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:38,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:39:38,917 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T20:39:38,922 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:38,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:39:38,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T20:39:38,928 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:38,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:39:38,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T20:39:38,934 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:38,936 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:39:38,936 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,942 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,944 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,952 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,953 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,958 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:39:38,962 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:39:38,966 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:39:38,968 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73282520, jitterRate=0.09199464321136475}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:39:38,974 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731357578866Initializing all the Stores at 1731357578868 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731357578869 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357578869Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357578870 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357578870Cleaning up temporary data from old regions at 1731357578953 (+83 ms)Region opened successfully at 1731357578974 (+21 ms) 2024-11-11T20:39:38,975 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T20:39:39,005 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5483a790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:39:39,034 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T20:39:39,045 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T20:39:39,045 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T20:39:39,047 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:39:39,049 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T20:39:39,053 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T20:39:39,053 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T20:39:39,078 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T20:39:39,087 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T20:39:39,088 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T20:39:39,091 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T20:39:39,092 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T20:39:39,093 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T20:39:39,095 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T20:39:39,099 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T20:39:39,100 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T20:39:39,102 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T20:39:39,103 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T20:39:39,119 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T20:39:39,121 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T20:39:39,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:39:39,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:39,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:39:39,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:39,128 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=51ca66f7ee3c,35885,1731357576039, sessionid=0x1003083643e0000, setting cluster-up flag (Was=false) 2024-11-11T20:39:39,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:39,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:39,143 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T20:39:39,145 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:39,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:39,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:39,154 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T20:39:39,156 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:39,165 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T20:39:39,196 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(746): ClusterId : 7eb98061-214b-40f9-8a48-d23c5a1d1708 2024-11-11T20:39:39,199 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:39:39,204 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:39:39,205 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:39:39,210 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:39:39,211 DEBUG [RS:0;51ca66f7ee3c:44271 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a05fcbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:39:39,225 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;51ca66f7ee3c:44271 2024-11-11T20:39:39,228 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:39:39,228 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:39:39,229 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:39:39,232 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,35885,1731357576039 with port=44271, startcode=1731357576703 2024-11-11T20:39:39,246 DEBUG [RS:0;51ca66f7ee3c:44271 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:39:39,252 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:39:39,264 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T20:39:39,273 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T20:39:39,280 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 51ca66f7ee3c,35885,1731357576039 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T20:39:39,285 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55219, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:39:39,291 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:39:39,291 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:39:39,291 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35885 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:39,292 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:39:39,292 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:39:39,292 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/51ca66f7ee3c:0, corePoolSize=10, maxPoolSize=10 2024-11-11T20:39:39,292 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,292 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:39:39,293 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,299 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731357609299 2024-11-11T20:39:39,299 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:39:39,300 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T20:39:39,301 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T20:39:39,302 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T20:39:39,306 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T20:39:39,306 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T20:39:39,307 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T20:39:39,307 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T20:39:39,308 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,310 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:39,310 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T20:39:39,313 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T20:39:39,314 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T20:39:39,315 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T20:39:39,323 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-11T20:39:39,324 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T20:39:39,325 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T20:39:39,325 WARN [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-11T20:39:39,329 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731357579326,5,FailOnTimeoutGroup] 2024-11-11T20:39:39,330 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731357579329,5,FailOnTimeoutGroup] 2024-11-11T20:39:39,330 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,330 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T20:39:39,332 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,333 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741831_1007 (size=1321) 2024-11-11T20:39:39,426 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,35885,1731357576039 with port=44271, startcode=1731357576703 2024-11-11T20:39:39,429 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35885 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:39,431 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35885 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:39,440 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30 2024-11-11T20:39:39,440 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38869 2024-11-11T20:39:39,440 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:39:39,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:39:39,445 DEBUG [RS:0;51ca66f7ee3c:44271 {}] zookeeper.ZKUtil(111): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:39,445 WARN [RS:0;51ca66f7ee3c:44271 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:39:39,445 INFO [RS:0;51ca66f7ee3c:44271 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:39:39,445 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/WALs/51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:39,447 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,44271,1731357576703] 2024-11-11T20:39:39,469 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:39:39,487 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:39:39,494 INFO [RS:0;51ca66f7ee3c:44271 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:39:39,494 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,495 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:39:39,501 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:39:39,503 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,503 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,503 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,504 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,504 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,504 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,504 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:39:39,505 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,505 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,505 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,505 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,505 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,506 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:39:39,506 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:39:39,506 DEBUG [RS:0;51ca66f7ee3c:44271 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:39:39,508 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,508 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,508 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,508 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,508 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,509 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,44271,1731357576703-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:39:39,531 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:39:39,533 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,44271,1731357576703-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,534 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,534 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.Replication(171): 51ca66f7ee3c,44271,1731357576703 started 2024-11-11T20:39:39,556 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:39,557 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,44271,1731357576703, RpcServer on 51ca66f7ee3c/172.17.0.2:44271, sessionid=0x1003083643e0001 2024-11-11T20:39:39,558 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:39:39,558 DEBUG [RS:0;51ca66f7ee3c:44271 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:39,558 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,44271,1731357576703' 2024-11-11T20:39:39,558 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:39:39,560 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:39:39,561 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:39:39,561 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:39:39,562 DEBUG [RS:0;51ca66f7ee3c:44271 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:39,562 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,44271,1731357576703' 2024-11-11T20:39:39,562 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:39:39,563 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:39:39,564 DEBUG [RS:0;51ca66f7ee3c:44271 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:39:39,565 INFO [RS:0;51ca66f7ee3c:44271 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:39:39,565 INFO [RS:0;51ca66f7ee3c:44271 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:39:39,671 INFO [RS:0;51ca66f7ee3c:44271 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:39:39,674 INFO [RS:0;51ca66f7ee3c:44271 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C44271%2C1731357576703, suffix=, logDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/WALs/51ca66f7ee3c,44271,1731357576703, archiveDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/oldWALs, maxLogs=32 2024-11-11T20:39:39,689 DEBUG [RS:0;51ca66f7ee3c:44271 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/WALs/51ca66f7ee3c,44271,1731357576703/51ca66f7ee3c%2C44271%2C1731357576703.1731357579676, exclude list is [], retry=0 2024-11-11T20:39:39,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38937,DS-744ed6d9-ac4c-4242-a538-2c1cc65f0abb,DISK] 2024-11-11T20:39:39,696 INFO [RS:0;51ca66f7ee3c:44271 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/WALs/51ca66f7ee3c,44271,1731357576703/51ca66f7ee3c%2C44271%2C1731357576703.1731357579676 2024-11-11T20:39:39,697 DEBUG [RS:0;51ca66f7ee3c:44271 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507)] 2024-11-11T20:39:39,743 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T20:39:39,743 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30 2024-11-11T20:39:39,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741833_1009 (size=32) 2024-11-11T20:39:40,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:39:40,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T20:39:40,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:39:40,170 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:40,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:39:40,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T20:39:40,175 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:39:40,175 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:40,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:39:40,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T20:39:40,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:39:40,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:40,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:39:40,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T20:39:40,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:39:40,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:40,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:39:40,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:39:40,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740 2024-11-11T20:39:40,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740 2024-11-11T20:39:40,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:39:40,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:39:40,190 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:39:40,192 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:39:40,196 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:39:40,197 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64264466, jitterRate=-0.042384833097457886}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:39:40,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731357580162Initializing all the Stores at 1731357580165 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731357580165Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731357580166 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357580166Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731357580166Cleaning up temporary data from old regions at 1731357580189 (+23 ms)Region opened successfully at 1731357580199 (+10 ms) 2024-11-11T20:39:40,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:39:40,200 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:39:40,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:39:40,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:39:40,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:39:40,201 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:39:40,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731357580199Disabling compacts and flushes for region at 1731357580199Disabling writes for close at 1731357580200 (+1 ms)Writing region close event to WAL at 1731357580201 (+1 ms)Closed at 1731357580201 2024-11-11T20:39:40,204 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:39:40,204 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T20:39:40,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T20:39:40,217 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:39:40,220 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T20:39:40,373 DEBUG [51ca66f7ee3c:35885 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T20:39:40,388 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:40,394 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,44271,1731357576703, state=OPENING 2024-11-11T20:39:40,398 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T20:39:40,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:40,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:39:40,400 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:39:40,400 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:39:40,402 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:39:40,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:39:40,577 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T20:39:40,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57733, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T20:39:40,590 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T20:39:40,590 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:39:40,590 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T20:39:40,594 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C44271%2C1731357576703.meta, suffix=.meta, logDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/WALs/51ca66f7ee3c,44271,1731357576703, archiveDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/oldWALs, maxLogs=32 2024-11-11T20:39:40,608 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/WALs/51ca66f7ee3c,44271,1731357576703/51ca66f7ee3c%2C44271%2C1731357576703.meta.1731357580595.meta, exclude list is [], retry=0 2024-11-11T20:39:40,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38937,DS-744ed6d9-ac4c-4242-a538-2c1cc65f0abb,DISK] 2024-11-11T20:39:40,615 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/WALs/51ca66f7ee3c,44271,1731357576703/51ca66f7ee3c%2C44271%2C1731357576703.meta.1731357580595.meta 2024-11-11T20:39:40,616 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507)] 2024-11-11T20:39:40,616 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:39:40,619 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T20:39:40,622 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T20:39:40,627 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T20:39:40,632 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T20:39:40,633 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:39:40,633 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T20:39:40,634 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T20:39:40,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T20:39:40,638 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:39:40,638 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:40,639 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:39:40,639 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T20:39:40,641 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:39:40,641 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:40,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:39:40,642 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T20:39:40,643 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:39:40,643 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:40,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:39:40,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T20:39:40,645 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:39:40,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:40,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:39:40,646 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:39:40,647 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740 2024-11-11T20:39:40,650 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740 2024-11-11T20:39:40,652 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:39:40,653 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:39:40,653 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:39:40,656 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:39:40,658 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64377791, jitterRate=-0.0406961590051651}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:39:40,658 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T20:39:40,660 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731357580634Writing region info on filesystem at 1731357580634Initializing all the Stores at 1731357580636 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731357580636Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731357580637 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357580637Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731357580637Cleaning up temporary data from old regions at 1731357580653 (+16 ms)Running coprocessor post-open hooks at 1731357580658 (+5 ms)Region opened successfully at 1731357580660 (+2 ms) 2024-11-11T20:39:40,668 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731357580568 2024-11-11T20:39:40,678 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T20:39:40,678 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T20:39:40,680 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:40,682 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,44271,1731357576703, state=OPEN 2024-11-11T20:39:40,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:39:40,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:39:40,686 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:39:40,686 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:39:40,686 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:40,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T20:39:40,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,44271,1731357576703 in 284 msec 2024-11-11T20:39:40,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T20:39:40,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 484 msec 2024-11-11T20:39:40,702 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:39:40,702 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T20:39:40,719 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:40,720 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:40,737 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:40,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56553, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:40,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5610 sec 2024-11-11T20:39:40,759 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731357580759, completionTime=-1 2024-11-11T20:39:40,761 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T20:39:40,762 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:39:40,786 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T20:39:40,786 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731357640786 2024-11-11T20:39:40,786 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731357700786 2024-11-11T20:39:40,787 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-11T20:39:40,789 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,35885,1731357576039-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:40,790 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,35885,1731357576039-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:40,790 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,35885,1731357576039-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:40,792 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-51ca66f7ee3c:35885, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:40,792 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:40,793 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:40,798 DEBUG [master/51ca66f7ee3c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T20:39:40,829 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.985sec 2024-11-11T20:39:40,830 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T20:39:40,831 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T20:39:40,833 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T20:39:40,833 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T20:39:40,834 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T20:39:40,835 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,35885,1731357576039-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:39:40,835 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,35885,1731357576039-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T20:39:40,845 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T20:39:40,846 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T20:39:40,847 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,35885,1731357576039-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:39:40,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:40,912 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T20:39:40,912 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T20:39:40,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:40,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:40,924 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7aa7eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:40,934 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:40,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:40,937 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:40,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:40,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:41,004 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=212, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=259, ProcessCount=11, AvailableMemoryMB=5162 2024-11-11T20:39:41,015 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:39:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:41,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23562587 2024-11-11T20:39:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:39:41,028 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:39:41,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:39:41,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:39:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:39:41,043 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:39:41,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T20:39:41,045 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:41,048 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:39:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:39:41,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741835_1011 (size=959) 2024-11-11T20:39:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:39:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:39:41,468 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30 2024-11-11T20:39:41,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741836_1012 (size=53) 2024-11-11T20:39:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:39:41,880 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:39:41,880 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fedb58c88f25aed82b02e36155b97788, disabling compactions & flushes 2024-11-11T20:39:41,881 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:41,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:41,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. after waiting 0 ms 2024-11-11T20:39:41,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:41,881 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:41,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fedb58c88f25aed82b02e36155b97788: Waiting for close lock at 1731357581880Disabling compacts and flushes for region at 1731357581880Disabling writes for close at 1731357581881 (+1 ms)Writing region close event to WAL at 1731357581881Closed at 1731357581881 2024-11-11T20:39:41,886 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:39:41,895 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731357581887"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731357581887"}]},"ts":"1731357581887"} 2024-11-11T20:39:41,900 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:39:41,902 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:39:41,904 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357581902"}]},"ts":"1731357581902"} 2024-11-11T20:39:41,908 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:39:41,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fedb58c88f25aed82b02e36155b97788, ASSIGN}] 2024-11-11T20:39:41,913 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fedb58c88f25aed82b02e36155b97788, ASSIGN 2024-11-11T20:39:41,915 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fedb58c88f25aed82b02e36155b97788, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44271,1731357576703; forceNewPlan=false, retain=false 2024-11-11T20:39:42,067 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fedb58c88f25aed82b02e36155b97788, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:42,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fedb58c88f25aed82b02e36155b97788, ASSIGN because future has completed 2024-11-11T20:39:42,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:39:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:39:42,242 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:42,242 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:39:42,243 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,243 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:39:42,243 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,243 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,246 INFO [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,248 INFO [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fedb58c88f25aed82b02e36155b97788 columnFamilyName A 2024-11-11T20:39:42,248 DEBUG [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:42,249 INFO [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] regionserver.HStore(327): Store=fedb58c88f25aed82b02e36155b97788/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:39:42,249 INFO [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,251 INFO [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fedb58c88f25aed82b02e36155b97788 columnFamilyName B 2024-11-11T20:39:42,251 DEBUG [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:42,252 INFO [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] regionserver.HStore(327): Store=fedb58c88f25aed82b02e36155b97788/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:39:42,252 INFO [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,254 INFO [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fedb58c88f25aed82b02e36155b97788 columnFamilyName C 2024-11-11T20:39:42,255 DEBUG [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:39:42,255 INFO [StoreOpener-fedb58c88f25aed82b02e36155b97788-1 {}] regionserver.HStore(327): Store=fedb58c88f25aed82b02e36155b97788/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:39:42,255 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,257 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,258 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,260 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,260 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,261 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:39:42,263 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,266 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:39:42,267 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fedb58c88f25aed82b02e36155b97788; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58977936, jitterRate=-0.12116026878356934}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:39:42,267 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:42,268 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fedb58c88f25aed82b02e36155b97788: Running coprocessor pre-open hook at 1731357582243Writing region info on filesystem at 1731357582243Initializing all the Stores at 1731357582245 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357582245Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357582245Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357582245Cleaning up temporary data from old regions at 1731357582260 (+15 ms)Running coprocessor post-open hooks at 1731357582267 (+7 ms)Region opened successfully at 1731357582268 (+1 ms) 2024-11-11T20:39:42,270 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., pid=6, masterSystemTime=1731357582232 2024-11-11T20:39:42,273 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:42,273 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:42,274 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fedb58c88f25aed82b02e36155b97788, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:42,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:39:42,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T20:39:42,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 in 203 msec 2024-11-11T20:39:42,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T20:39:42,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fedb58c88f25aed82b02e36155b97788, ASSIGN in 374 msec 2024-11-11T20:39:42,290 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:39:42,290 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357582290"}]},"ts":"1731357582290"} 2024-11-11T20:39:42,294 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:39:42,295 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:39:42,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2590 sec 2024-11-11T20:39:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:39:43,203 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:39:43,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775e68a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,216 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d34895b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,219 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50806732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,222 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,228 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:39:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,35885,1731357576039 2024-11-11T20:39:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d4a6a92 2024-11-11T20:39:43,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:39:43,233 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:39:43,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,239 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e11900a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,241 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,241 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,242 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca0b5f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,246 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387dfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,249 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25175df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,253 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e6a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,257 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,260 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f338c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,263 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10112b3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,266 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c3f9496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55bbabde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,272 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b71435d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,275 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4425cde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,278 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79aeff1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,280 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64994b9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,283 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f551be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189704a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c9a155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:39:43,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:39:43,288 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:39:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:39:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:39:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7299e0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:39:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:39:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:39:43,290 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:39:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39568e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:39:43,295 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,295 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,295 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,296 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,296 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,295 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,297 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:39:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,299 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,302 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:39:43,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T20:39:43,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:39:43,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:39:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:39:43,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:39:43,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:39:43,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:39:43,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:39:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:39:43,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:39:43,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:39:43,315 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:39:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:39:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:39:43,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:39:43,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:39:43,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:39:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:39:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:43,422 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:43,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:39:43,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:39:43,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:43,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:43,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:43,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:43,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/ab41baa101484bb4b502095ed23a62ac is 50, key is test_row_0/A:col10/1731357583409/Put/seqid=0 2024-11-11T20:39:43,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741837_1013 (size=12001) 2024-11-11T20:39:43,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/ab41baa101484bb4b502095ed23a62ac 2024-11-11T20:39:43,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357643593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:39:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357643603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357643622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357643623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357643625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:39:43,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:43,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:43,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:43,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:43,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/d378195f06d54de9b2b5cf9cf73eeff9 is 50, key is test_row_0/B:col10/1731357583409/Put/seqid=0 2024-11-11T20:39:43,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741838_1014 (size=12001) 2024-11-11T20:39:43,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/d378195f06d54de9b2b5cf9cf73eeff9 2024-11-11T20:39:43,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357643732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357643741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357643763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357643764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357643764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:43,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/0ad0518b696d420ead376e1d9576dfd3 is 50, key is test_row_0/C:col10/1731357583409/Put/seqid=0 2024-11-11T20:39:43,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741839_1015 (size=12001) 2024-11-11T20:39:43,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/0ad0518b696d420ead376e1d9576dfd3 2024-11-11T20:39:43,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:39:43,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:43,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:43,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/ab41baa101484bb4b502095ed23a62ac as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ab41baa101484bb4b502095ed23a62ac 2024-11-11T20:39:43,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:43,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:43,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:43,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ab41baa101484bb4b502095ed23a62ac, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:39:43,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/d378195f06d54de9b2b5cf9cf73eeff9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d378195f06d54de9b2b5cf9cf73eeff9 2024-11-11T20:39:43,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d378195f06d54de9b2b5cf9cf73eeff9, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:39:43,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/0ad0518b696d420ead376e1d9576dfd3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/0ad0518b696d420ead376e1d9576dfd3 2024-11-11T20:39:43,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/0ad0518b696d420ead376e1d9576dfd3, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:39:43,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for fedb58c88f25aed82b02e36155b97788 in 501ms, sequenceid=13, compaction requested=false 2024-11-11T20:39:43,937 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T20:39:43,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:39:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:43,963 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:43,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T20:39:43,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/d906ad94c4714b4ea866c2a33456ceff is 50, key is test_row_0/A:col10/1731357583607/Put/seqid=0 2024-11-11T20:39:44,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357643986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357643994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357643997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357644000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357644006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:39:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:44,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:44,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:44,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:44,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741840_1016 (size=14341) 2024-11-11T20:39:44,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/d906ad94c4714b4ea866c2a33456ceff 2024-11-11T20:39:44,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/dad6c08c868d4d2986c5c2a39e4d3641 is 50, key is test_row_0/B:col10/1731357583607/Put/seqid=0 2024-11-11T20:39:44,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741841_1017 (size=12001) 2024-11-11T20:39:44,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/dad6c08c868d4d2986c5c2a39e4d3641 2024-11-11T20:39:44,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357644112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357644113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357644113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357644124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357644123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/740527e569b242a6a329f62e91e51e83 is 50, key is test_row_0/C:col10/1731357583607/Put/seqid=0 2024-11-11T20:39:44,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:39:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:44,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:44,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:44,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741842_1018 (size=12001) 2024-11-11T20:39:44,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/740527e569b242a6a329f62e91e51e83 2024-11-11T20:39:44,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/d906ad94c4714b4ea866c2a33456ceff as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d906ad94c4714b4ea866c2a33456ceff 2024-11-11T20:39:44,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d906ad94c4714b4ea866c2a33456ceff, entries=200, sequenceid=40, filesize=14.0 K 2024-11-11T20:39:44,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/dad6c08c868d4d2986c5c2a39e4d3641 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/dad6c08c868d4d2986c5c2a39e4d3641 2024-11-11T20:39:44,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/dad6c08c868d4d2986c5c2a39e4d3641, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T20:39:44,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/740527e569b242a6a329f62e91e51e83 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/740527e569b242a6a329f62e91e51e83 2024-11-11T20:39:44,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/740527e569b242a6a329f62e91e51e83, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T20:39:44,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for fedb58c88f25aed82b02e36155b97788 in 356ms, sequenceid=40, compaction requested=false 2024-11-11T20:39:44,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:44,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:39:44,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:44,330 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-11T20:39:44,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/b29ac9133b9740d0993018fc9400919c is 50, key is test_row_0/A:col10/1731357583999/Put/seqid=0 2024-11-11T20:39:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:44,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:44,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741843_1019 (size=12001) 2024-11-11T20:39:44,399 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/b29ac9133b9740d0993018fc9400919c 2024-11-11T20:39:44,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5692ef02aa26482592f88368f865e48a is 50, key is test_row_0/B:col10/1731357583999/Put/seqid=0 2024-11-11T20:39:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:39:44,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741844_1020 (size=12001) 2024-11-11T20:39:44,454 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5692ef02aa26482592f88368f865e48a 2024-11-11T20:39:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/77e6160d4db24c9c85bb0eae8775997e is 50, key is test_row_0/C:col10/1731357583999/Put/seqid=0 2024-11-11T20:39:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357644464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357644481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357644487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357644492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357644488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741845_1021 (size=12001) 2024-11-11T20:39:44,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357644602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357644602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357644613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357644614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357644614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357644814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357644814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357644832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357644832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357644832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:44,910 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/77e6160d4db24c9c85bb0eae8775997e 2024-11-11T20:39:44,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/b29ac9133b9740d0993018fc9400919c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b29ac9133b9740d0993018fc9400919c 2024-11-11T20:39:44,937 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b29ac9133b9740d0993018fc9400919c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:39:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5692ef02aa26482592f88368f865e48a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5692ef02aa26482592f88368f865e48a 2024-11-11T20:39:44,958 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5692ef02aa26482592f88368f865e48a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:39:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/77e6160d4db24c9c85bb0eae8775997e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/77e6160d4db24c9c85bb0eae8775997e 2024-11-11T20:39:44,977 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/77e6160d4db24c9c85bb0eae8775997e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:39:44,979 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for fedb58c88f25aed82b02e36155b97788 in 649ms, sequenceid=49, compaction requested=true 2024-11-11T20:39:44,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:44,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:44,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T20:39:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T20:39:44,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T20:39:44,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6610 sec 2024-11-11T20:39:44,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6950 sec 2024-11-11T20:39:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:45,158 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:45,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-11T20:39:45,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/dd9cc8030f574e59a078c5c5d3ad2930 is 50, key is test_row_0/A:col10/1731357585147/Put/seqid=0 2024-11-11T20:39:45,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357645177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357645181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357645181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741846_1022 (size=12001) 2024-11-11T20:39:45,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357645227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/dd9cc8030f574e59a078c5c5d3ad2930 2024-11-11T20:39:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357645227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/ee11eae90cce4df992bd3c5c9f21967d is 50, key is test_row_0/B:col10/1731357585147/Put/seqid=0 2024-11-11T20:39:45,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741847_1023 (size=12001) 2024-11-11T20:39:45,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/ee11eae90cce4df992bd3c5c9f21967d 2024-11-11T20:39:45,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/4173c0af9af84d928b9786df9c65dace is 50, key is test_row_0/C:col10/1731357585147/Put/seqid=0 2024-11-11T20:39:45,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357645345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357645345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357645345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357645352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741848_1024 (size=12001) 2024-11-11T20:39:45,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357645363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:39:45,462 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:39:45,464 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T20:39:45,465 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:39:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T20:39:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:39:45,470 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:39:45,472 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:39:45,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:39:45,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357645561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357645561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357645561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357645562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357645571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:39:45,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:39:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:45,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:45,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/4173c0af9af84d928b9786df9c65dace 2024-11-11T20:39:45,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/dd9cc8030f574e59a078c5c5d3ad2930 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dd9cc8030f574e59a078c5c5d3ad2930 2024-11-11T20:39:45,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:39:45,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:45,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:39:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:45,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dd9cc8030f574e59a078c5c5d3ad2930, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T20:39:45,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/ee11eae90cce4df992bd3c5c9f21967d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ee11eae90cce4df992bd3c5c9f21967d 2024-11-11T20:39:45,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ee11eae90cce4df992bd3c5c9f21967d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T20:39:45,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/4173c0af9af84d928b9786df9c65dace as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4173c0af9af84d928b9786df9c65dace 2024-11-11T20:39:45,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4173c0af9af84d928b9786df9c65dace, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T20:39:45,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for fedb58c88f25aed82b02e36155b97788 in 676ms, sequenceid=79, compaction requested=true 2024-11-11T20:39:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:45,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:39:45,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:45,839 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:39:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:45,840 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:39:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:45,853 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:45,855 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/B is initiating minor compaction (all files) 2024-11-11T20:39:45,856 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/B in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:45,859 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d378195f06d54de9b2b5cf9cf73eeff9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/dad6c08c868d4d2986c5c2a39e4d3641, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5692ef02aa26482592f88368f865e48a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ee11eae90cce4df992bd3c5c9f21967d] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=46.9 K 2024-11-11T20:39:45,861 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:45,861 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/A is initiating minor compaction (all files) 2024-11-11T20:39:45,861 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d378195f06d54de9b2b5cf9cf73eeff9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731357583398 2024-11-11T20:39:45,861 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/A in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:45,862 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ab41baa101484bb4b502095ed23a62ac, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d906ad94c4714b4ea866c2a33456ceff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b29ac9133b9740d0993018fc9400919c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dd9cc8030f574e59a078c5c5d3ad2930] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=49.2 K 2024-11-11T20:39:45,864 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting dad6c08c868d4d2986c5c2a39e4d3641, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731357583607 2024-11-11T20:39:45,864 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab41baa101484bb4b502095ed23a62ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731357583398 2024-11-11T20:39:45,865 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d906ad94c4714b4ea866c2a33456ceff, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731357583598 2024-11-11T20:39:45,869 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5692ef02aa26482592f88368f865e48a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357583982 2024-11-11T20:39:45,869 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b29ac9133b9740d0993018fc9400919c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357583982 2024-11-11T20:39:45,871 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd9cc8030f574e59a078c5c5d3ad2930, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731357584486 2024-11-11T20:39:45,872 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ee11eae90cce4df992bd3c5c9f21967d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731357584486 2024-11-11T20:39:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:45,906 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:45,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:39:45,941 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#B#compaction#12 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:45,941 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#A#compaction#13 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:45,942 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e09f90fb0aac402d99dd8d8d5ad1f6ae is 50, key is test_row_0/A:col10/1731357585147/Put/seqid=0 2024-11-11T20:39:45,943 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/d4ec22421ebb4f759a1f816c9ca9b502 is 50, key is test_row_0/B:col10/1731357585147/Put/seqid=0 2024-11-11T20:39:45,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/5df1d43025b7456a8c5361a743a8ad22 is 50, key is test_row_0/A:col10/1731357585189/Put/seqid=0 2024-11-11T20:39:45,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:39:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:45,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:45,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:45,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:45,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:45,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741850_1026 (size=12139) 2024-11-11T20:39:46,005 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e09f90fb0aac402d99dd8d8d5ad1f6ae as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e09f90fb0aac402d99dd8d8d5ad1f6ae 2024-11-11T20:39:46,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741849_1025 (size=12139) 2024-11-11T20:39:46,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741851_1027 (size=14341) 2024-11-11T20:39:46,059 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/A of fedb58c88f25aed82b02e36155b97788 into e09f90fb0aac402d99dd8d8d5ad1f6ae(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:46,059 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:46,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/5df1d43025b7456a8c5361a743a8ad22 2024-11-11T20:39:46,064 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/A, priority=12, startTime=1731357585836; duration=0sec 2024-11-11T20:39:46,065 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:46,065 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:A 2024-11-11T20:39:46,065 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:46,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357646025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357646042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,081 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:46,081 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/C is initiating minor compaction (all files) 2024-11-11T20:39:46,082 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/C in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,082 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/0ad0518b696d420ead376e1d9576dfd3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/740527e569b242a6a329f62e91e51e83, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/77e6160d4db24c9c85bb0eae8775997e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4173c0af9af84d928b9786df9c65dace] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=46.9 K 2024-11-11T20:39:46,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357646060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,083 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ad0518b696d420ead376e1d9576dfd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731357583398 2024-11-11T20:39:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,085 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 740527e569b242a6a329f62e91e51e83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731357583607 2024-11-11T20:39:46,086 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77e6160d4db24c9c85bb0eae8775997e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357583982 2024-11-11T20:39:46,088 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4173c0af9af84d928b9786df9c65dace, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731357584486 2024-11-11T20:39:46,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357646074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/e54b40f486214c0a8fee194b9ef9ec3b is 50, key is test_row_0/B:col10/1731357585189/Put/seqid=0 2024-11-11T20:39:46,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357646079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:39:46,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:46,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:39:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,157 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:39:46,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741852_1028 (size=12001) 2024-11-11T20:39:46,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/e54b40f486214c0a8fee194b9ef9ec3b 2024-11-11T20:39:46,164 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#C#compaction#16 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:46,167 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/ba0b7e5e03074f05a83aeaa15aa3c63e is 50, key is test_row_0/C:col10/1731357585147/Put/seqid=0 2024-11-11T20:39:46,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/c142e1f468cd446791e655bd238daefa is 50, key is test_row_0/C:col10/1731357585189/Put/seqid=0 2024-11-11T20:39:46,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357646181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357646181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357646191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357646202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357646211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741853_1029 (size=12139) 2024-11-11T20:39:46,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:39:46,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:46,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,270 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/ba0b7e5e03074f05a83aeaa15aa3c63e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/ba0b7e5e03074f05a83aeaa15aa3c63e 2024-11-11T20:39:46,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741854_1030 (size=12001) 2024-11-11T20:39:46,291 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/C of fedb58c88f25aed82b02e36155b97788 into ba0b7e5e03074f05a83aeaa15aa3c63e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:46,291 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:46,292 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/C, priority=12, startTime=1731357585840; duration=0sec 2024-11-11T20:39:46,292 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:46,292 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:C 2024-11-11T20:39:46,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357646412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:39:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357646412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357646412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357646412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,440 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/d4ec22421ebb4f759a1f816c9ca9b502 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d4ec22421ebb4f759a1f816c9ca9b502 2024-11-11T20:39:46,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357646431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,452 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T20:39:46,454 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T20:39:46,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:39:46,457 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T20:39:46,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T20:39:46,457 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T20:39:46,459 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T20:39:46,459 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T20:39:46,459 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/B of fedb58c88f25aed82b02e36155b97788 into d4ec22421ebb4f759a1f816c9ca9b502(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:46,459 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:46,459 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/B, priority=12, startTime=1731357585839; duration=0sec 2024-11-11T20:39:46,460 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:46,460 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:B 2024-11-11T20:39:46,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:39:46,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:39:46,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/c142e1f468cd446791e655bd238daefa 2024-11-11T20:39:46,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/5df1d43025b7456a8c5361a743a8ad22 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/5df1d43025b7456a8c5361a743a8ad22 2024-11-11T20:39:46,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/5df1d43025b7456a8c5361a743a8ad22, entries=200, sequenceid=91, filesize=14.0 K 2024-11-11T20:39:46,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357646721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357646722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/e54b40f486214c0a8fee194b9ef9ec3b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/e54b40f486214c0a8fee194b9ef9ec3b 2024-11-11T20:39:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:39:46,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:46,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:46,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357646734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357646734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/e54b40f486214c0a8fee194b9ef9ec3b, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T20:39:46,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/c142e1f468cd446791e655bd238daefa as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/c142e1f468cd446791e655bd238daefa 2024-11-11T20:39:46,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357646751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:46,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/c142e1f468cd446791e655bd238daefa, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T20:39:46,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fedb58c88f25aed82b02e36155b97788 in 862ms, sequenceid=91, compaction requested=false 2024-11-11T20:39:46,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:46,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:39:46,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:46,887 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:39:46,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/721629911c654800a52c5b9a0d3d562a is 50, key is test_row_0/A:col10/1731357586062/Put/seqid=0 2024-11-11T20:39:46,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741855_1031 (size=12001) 2024-11-11T20:39:46,929 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/721629911c654800a52c5b9a0d3d562a 2024-11-11T20:39:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/fb248d619edc4d6087b60e951e3a6a9a is 50, key is test_row_0/B:col10/1731357586062/Put/seqid=0 2024-11-11T20:39:46,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741856_1032 (size=12001) 2024-11-11T20:39:46,968 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/fb248d619edc4d6087b60e951e3a6a9a 2024-11-11T20:39:46,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/abb5853dab434741a6411ca8aab091fd is 50, key is test_row_0/C:col10/1731357586062/Put/seqid=0 2024-11-11T20:39:47,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741857_1033 (size=12001) 2024-11-11T20:39:47,015 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/abb5853dab434741a6411ca8aab091fd 2024-11-11T20:39:47,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/721629911c654800a52c5b9a0d3d562a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/721629911c654800a52c5b9a0d3d562a 2024-11-11T20:39:47,046 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/721629911c654800a52c5b9a0d3d562a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T20:39:47,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T20:39:47,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/fb248d619edc4d6087b60e951e3a6a9a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/fb248d619edc4d6087b60e951e3a6a9a 2024-11-11T20:39:47,069 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/fb248d619edc4d6087b60e951e3a6a9a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T20:39:47,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/abb5853dab434741a6411ca8aab091fd as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/abb5853dab434741a6411ca8aab091fd 2024-11-11T20:39:47,084 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/abb5853dab434741a6411ca8aab091fd, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T20:39:47,087 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for fedb58c88f25aed82b02e36155b97788 in 200ms, sequenceid=118, compaction requested=true 2024-11-11T20:39:47,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:47,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:47,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T20:39:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T20:39:47,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T20:39:47,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6190 sec 2024-11-11T20:39:47,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.6340 sec 2024-11-11T20:39:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:47,259 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:47,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:39:47,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/afd6353f69f5419d84f37ba0033cf6c3 is 50, key is test_row_0/A:col10/1731357587252/Put/seqid=0 2024-11-11T20:39:47,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741858_1034 (size=12101) 2024-11-11T20:39:47,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/afd6353f69f5419d84f37ba0033cf6c3 2024-11-11T20:39:47,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/836966131bf34b9fbc319e2506100bfe is 50, key is test_row_0/B:col10/1731357587252/Put/seqid=0 2024-11-11T20:39:47,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741859_1035 (size=12101) 2024-11-11T20:39:47,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357647373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357647375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357647379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357647383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357647386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357647492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357647492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357647501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357647502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357647492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:39:47,621 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:39:47,621 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:39:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T20:39:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:39:47,624 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:39:47,626 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:39:47,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:39:47,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357647711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357647711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357647711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357647711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/836966131bf34b9fbc319e2506100bfe 2024-11-11T20:39:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:39:47,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357647733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:47,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/7b17715326254ead93cb1ad1bcb07f39 is 50, key is test_row_0/C:col10/1731357587252/Put/seqid=0 2024-11-11T20:39:47,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741860_1036 (size=12101) 2024-11-11T20:39:47,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/7b17715326254ead93cb1ad1bcb07f39 2024-11-11T20:39:47,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/afd6353f69f5419d84f37ba0033cf6c3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/afd6353f69f5419d84f37ba0033cf6c3 2024-11-11T20:39:47,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:39:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:47,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:47,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/afd6353f69f5419d84f37ba0033cf6c3, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T20:39:47,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/836966131bf34b9fbc319e2506100bfe as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/836966131bf34b9fbc319e2506100bfe 2024-11-11T20:39:47,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/836966131bf34b9fbc319e2506100bfe, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T20:39:47,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/7b17715326254ead93cb1ad1bcb07f39 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7b17715326254ead93cb1ad1bcb07f39 2024-11-11T20:39:47,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7b17715326254ead93cb1ad1bcb07f39, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T20:39:47,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for fedb58c88f25aed82b02e36155b97788 in 590ms, sequenceid=130, compaction requested=true 2024-11-11T20:39:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:39:47,851 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:39:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:47,851 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:39:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:47,853 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50582 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:47,853 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:47,853 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/B is initiating minor compaction (all files) 2024-11-11T20:39:47,853 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/A is initiating minor compaction (all files) 2024-11-11T20:39:47,853 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/A in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:47,853 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/B in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:47,854 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d4ec22421ebb4f759a1f816c9ca9b502, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/e54b40f486214c0a8fee194b9ef9ec3b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/fb248d619edc4d6087b60e951e3a6a9a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/836966131bf34b9fbc319e2506100bfe] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=47.1 K 2024-11-11T20:39:47,854 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e09f90fb0aac402d99dd8d8d5ad1f6ae, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/5df1d43025b7456a8c5361a743a8ad22, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/721629911c654800a52c5b9a0d3d562a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/afd6353f69f5419d84f37ba0033cf6c3] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=49.4 K 2024-11-11T20:39:47,855 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ec22421ebb4f759a1f816c9ca9b502, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731357584486 2024-11-11T20:39:47,855 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e09f90fb0aac402d99dd8d8d5ad1f6ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731357584486 2024-11-11T20:39:47,856 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting e54b40f486214c0a8fee194b9ef9ec3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731357585189 2024-11-11T20:39:47,857 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5df1d43025b7456a8c5361a743a8ad22, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731357585189 2024-11-11T20:39:47,857 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fb248d619edc4d6087b60e951e3a6a9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731357585977 2024-11-11T20:39:47,858 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721629911c654800a52c5b9a0d3d562a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731357585977 2024-11-11T20:39:47,860 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 836966131bf34b9fbc319e2506100bfe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357587252 2024-11-11T20:39:47,860 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd6353f69f5419d84f37ba0033cf6c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357587252 2024-11-11T20:39:47,899 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#A#compaction#24 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:47,900 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/6d93cc8439da45f997b01a2a6c2064c0 is 50, key is test_row_0/A:col10/1731357587252/Put/seqid=0 2024-11-11T20:39:47,904 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:47,906 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/02ab58dae98e4c5f8b2607425bb5297e is 50, key is test_row_0/B:col10/1731357587252/Put/seqid=0 2024-11-11T20:39:47,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741861_1037 (size=12375) 2024-11-11T20:39:47,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:39:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:47,939 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T20:39:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:39:47,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741862_1038 (size=12375) 2024-11-11T20:39:47,951 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/6d93cc8439da45f997b01a2a6c2064c0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/6d93cc8439da45f997b01a2a6c2064c0 2024-11-11T20:39:47,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/72043ec4ec7f4867bc6fe97baa48d709 is 50, key is test_row_0/A:col10/1731357587381/Put/seqid=0 2024-11-11T20:39:47,964 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/A of fedb58c88f25aed82b02e36155b97788 into 6d93cc8439da45f997b01a2a6c2064c0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:47,964 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:47,965 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/A, priority=12, startTime=1731357587851; duration=0sec 2024-11-11T20:39:47,965 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:47,965 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:A 2024-11-11T20:39:47,965 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:47,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741863_1039 (size=12151) 2024-11-11T20:39:47,969 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:47,969 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/C is initiating minor compaction (all files) 2024-11-11T20:39:47,969 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/C in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:47,970 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/ba0b7e5e03074f05a83aeaa15aa3c63e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/c142e1f468cd446791e655bd238daefa, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/abb5853dab434741a6411ca8aab091fd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7b17715326254ead93cb1ad1bcb07f39] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=47.1 K 2024-11-11T20:39:47,970 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0b7e5e03074f05a83aeaa15aa3c63e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731357584486 2024-11-11T20:39:47,971 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c142e1f468cd446791e655bd238daefa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731357585189 2024-11-11T20:39:47,972 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting abb5853dab434741a6411ca8aab091fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731357585977 2024-11-11T20:39:47,972 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b17715326254ead93cb1ad1bcb07f39, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357587252 2024-11-11T20:39:47,979 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/72043ec4ec7f4867bc6fe97baa48d709 2024-11-11T20:39:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/6ae0b4f21dad48a6a10e00894aa93b1e is 50, key is test_row_0/B:col10/1731357587381/Put/seqid=0 2024-11-11T20:39:48,008 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#C#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:48,009 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/69ed921a2fc74658b1afed47c3f6d605 is 50, key is test_row_0/C:col10/1731357587252/Put/seqid=0 2024-11-11T20:39:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:48,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:48,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741864_1040 (size=12151) 2024-11-11T20:39:48,050 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/6ae0b4f21dad48a6a10e00894aa93b1e 2024-11-11T20:39:48,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357648049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357648049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357648055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357648056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357648058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/df7dbc2d20964be4873581adc7409e69 is 50, key is test_row_0/C:col10/1731357587381/Put/seqid=0 2024-11-11T20:39:48,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741865_1041 (size=12375) 2024-11-11T20:39:48,102 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/69ed921a2fc74658b1afed47c3f6d605 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/69ed921a2fc74658b1afed47c3f6d605 2024-11-11T20:39:48,126 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/C of fedb58c88f25aed82b02e36155b97788 into 69ed921a2fc74658b1afed47c3f6d605(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:48,126 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:48,126 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/C, priority=12, startTime=1731357587851; duration=0sec 2024-11-11T20:39:48,126 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:48,126 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:C 2024-11-11T20:39:48,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741866_1042 (size=12151) 2024-11-11T20:39:48,145 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/df7dbc2d20964be4873581adc7409e69 2024-11-11T20:39:48,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/72043ec4ec7f4867bc6fe97baa48d709 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/72043ec4ec7f4867bc6fe97baa48d709 2024-11-11T20:39:48,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357648161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357648161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357648171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357648171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,180 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/72043ec4ec7f4867bc6fe97baa48d709, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T20:39:48,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357648180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/6ae0b4f21dad48a6a10e00894aa93b1e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6ae0b4f21dad48a6a10e00894aa93b1e 2024-11-11T20:39:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,200 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6ae0b4f21dad48a6a10e00894aa93b1e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T20:39:48,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/df7dbc2d20964be4873581adc7409e69 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/df7dbc2d20964be4873581adc7409e69 2024-11-11T20:39:48,215 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/df7dbc2d20964be4873581adc7409e69, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T20:39:48,222 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for fedb58c88f25aed82b02e36155b97788 in 283ms, sequenceid=155, compaction requested=false 2024-11-11T20:39:48,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:48,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T20:39:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T20:39:48,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T20:39:48,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 599 msec 2024-11-11T20:39:48,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 615 msec 2024-11-11T20:39:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:39:48,251 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:39:48,252 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:39:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T20:39:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:39:48,256 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:39:48,258 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:39:48,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:39:48,358 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/02ab58dae98e4c5f8b2607425bb5297e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/02ab58dae98e4c5f8b2607425bb5297e 2024-11-11T20:39:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:39:48,371 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/B of fedb58c88f25aed82b02e36155b97788 into 02ab58dae98e4c5f8b2607425bb5297e(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:48,371 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:48,371 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/B, priority=12, startTime=1731357587851; duration=0sec 2024-11-11T20:39:48,373 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:48,373 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:B 2024-11-11T20:39:48,373 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:48,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T20:39:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:48,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/04405569c3354719a5525ca2ca4a4987 is 50, key is test_row_0/A:col10/1731357588046/Put/seqid=0 2024-11-11T20:39:48,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741867_1043 (size=14541) 2024-11-11T20:39:48,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/04405569c3354719a5525ca2ca4a4987 2024-11-11T20:39:48,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:39:48,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:48,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/095caa38b12443ee8ac05b0392543c12 is 50, key is test_row_0/B:col10/1731357588046/Put/seqid=0 2024-11-11T20:39:48,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741868_1044 (size=12151) 2024-11-11T20:39:48,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/095caa38b12443ee8ac05b0392543c12 2024-11-11T20:39:48,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357648425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357648427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357648425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357648427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357648431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/7e56bef185364fd5b0aca9dc207037e0 is 50, key is test_row_0/C:col10/1731357588046/Put/seqid=0 2024-11-11T20:39:48,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741869_1045 (size=12151) 2024-11-11T20:39:48,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/7e56bef185364fd5b0aca9dc207037e0 2024-11-11T20:39:48,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/04405569c3354719a5525ca2ca4a4987 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/04405569c3354719a5525ca2ca4a4987 2024-11-11T20:39:48,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/04405569c3354719a5525ca2ca4a4987, entries=200, sequenceid=171, filesize=14.2 K 2024-11-11T20:39:48,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/095caa38b12443ee8ac05b0392543c12 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/095caa38b12443ee8ac05b0392543c12 2024-11-11T20:39:48,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/095caa38b12443ee8ac05b0392543c12, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T20:39:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/7e56bef185364fd5b0aca9dc207037e0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7e56bef185364fd5b0aca9dc207037e0 2024-11-11T20:39:48,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7e56bef185364fd5b0aca9dc207037e0, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T20:39:48,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for fedb58c88f25aed82b02e36155b97788 in 160ms, sequenceid=171, compaction requested=true 2024-11-11T20:39:48,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:48,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:39:48,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:48,533 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:39:48,534 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:39:48,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:39:48,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:48,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:39:48,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:48,536 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:39:48,536 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/B is initiating minor compaction (all files) 2024-11-11T20:39:48,536 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/B in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,536 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/02ab58dae98e4c5f8b2607425bb5297e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6ae0b4f21dad48a6a10e00894aa93b1e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/095caa38b12443ee8ac05b0392543c12] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=35.8 K 2024-11-11T20:39:48,539 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39067 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:39:48,539 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/A is initiating minor compaction (all files) 2024-11-11T20:39:48,539 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ab58dae98e4c5f8b2607425bb5297e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357587252 2024-11-11T20:39:48,539 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/A in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,539 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/6d93cc8439da45f997b01a2a6c2064c0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/72043ec4ec7f4867bc6fe97baa48d709, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/04405569c3354719a5525ca2ca4a4987] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=38.2 K 2024-11-11T20:39:48,540 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d93cc8439da45f997b01a2a6c2064c0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357587252 2024-11-11T20:39:48,540 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ae0b4f21dad48a6a10e00894aa93b1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731357587372 2024-11-11T20:39:48,541 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72043ec4ec7f4867bc6fe97baa48d709, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731357587372 2024-11-11T20:39:48,548 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 095caa38b12443ee8ac05b0392543c12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731357588046 2024-11-11T20:39:48,550 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04405569c3354719a5525ca2ca4a4987, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731357588046 2024-11-11T20:39:48,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:48,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T20:39:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:48,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e9f069db477e435f8dbddbbc20fc6499 is 50, key is test_row_0/A:col10/1731357588423/Put/seqid=0 2024-11-11T20:39:48,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:39:48,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:39:48,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:48,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,580 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:48,581 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/1cd729b47619419689899242b21aa003 is 50, key is test_row_0/B:col10/1731357588046/Put/seqid=0 2024-11-11T20:39:48,584 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#A#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:48,585 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/def87914f09e4b99b7dd618cc562ee2f is 50, key is test_row_0/A:col10/1731357588046/Put/seqid=0 2024-11-11T20:39:48,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741870_1046 (size=16931) 2024-11-11T20:39:48,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741871_1047 (size=12527) 2024-11-11T20:39:48,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741872_1048 (size=12527) 2024-11-11T20:39:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357648611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357648612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357648612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357648618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357648621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:39:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357648732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357648732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357648732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357648732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357648733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:39:48,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:48,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:48,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:39:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357648941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357648942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357648942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357648942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357648942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:48,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e9f069db477e435f8dbddbbc20fc6499 2024-11-11T20:39:49,007 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/1cd729b47619419689899242b21aa003 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/1cd729b47619419689899242b21aa003 2024-11-11T20:39:49,016 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/def87914f09e4b99b7dd618cc562ee2f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/def87914f09e4b99b7dd618cc562ee2f 2024-11-11T20:39:49,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5b127794e9084765af20f94619937661 is 50, key is test_row_0/B:col10/1731357588423/Put/seqid=0 2024-11-11T20:39:49,026 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fedb58c88f25aed82b02e36155b97788/B of fedb58c88f25aed82b02e36155b97788 into 1cd729b47619419689899242b21aa003(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:49,026 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:49,026 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/B, priority=13, startTime=1731357588534; duration=0sec 2024-11-11T20:39:49,026 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:49,026 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fedb58c88f25aed82b02e36155b97788/A of fedb58c88f25aed82b02e36155b97788 into def87914f09e4b99b7dd618cc562ee2f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:49,026 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:B 2024-11-11T20:39:49,026 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:49,026 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/A, priority=13, startTime=1731357588533; duration=0sec 2024-11-11T20:39:49,026 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:49,026 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:39:49,026 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:A 2024-11-11T20:39:49,030 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:39:49,030 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/C is initiating minor compaction (all files) 2024-11-11T20:39:49,030 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/C in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,031 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/69ed921a2fc74658b1afed47c3f6d605, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/df7dbc2d20964be4873581adc7409e69, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7e56bef185364fd5b0aca9dc207037e0] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=35.8 K 2024-11-11T20:39:49,032 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 69ed921a2fc74658b1afed47c3f6d605, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357587252 2024-11-11T20:39:49,032 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting df7dbc2d20964be4873581adc7409e69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731357587372 2024-11-11T20:39:49,034 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e56bef185364fd5b0aca9dc207037e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731357588046 2024-11-11T20:39:49,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:39:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741873_1049 (size=12151) 2024-11-11T20:39:49,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5b127794e9084765af20f94619937661 2024-11-11T20:39:49,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/fb5ce47e0d334e17adbc9ad0ab21d6a0 is 50, key is test_row_0/C:col10/1731357588423/Put/seqid=0 2024-11-11T20:39:49,063 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#C#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:49,064 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/feb037f937034510bc0cc78c599443a6 is 50, key is test_row_0/C:col10/1731357588046/Put/seqid=0 2024-11-11T20:39:49,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741874_1050 (size=12151) 2024-11-11T20:39:49,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741875_1051 (size=12527) 2024-11-11T20:39:49,109 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/feb037f937034510bc0cc78c599443a6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/feb037f937034510bc0cc78c599443a6 2024-11-11T20:39:49,120 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fedb58c88f25aed82b02e36155b97788/C of fedb58c88f25aed82b02e36155b97788 into feb037f937034510bc0cc78c599443a6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:49,120 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:49,120 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/C, priority=13, startTime=1731357588534; duration=0sec 2024-11-11T20:39:49,120 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:49,120 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:C 2024-11-11T20:39:49,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:39:49,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:49,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357649251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357649251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357649252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357649261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357649261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:39:49,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:49,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:39:49,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/fb5ce47e0d334e17adbc9ad0ab21d6a0 2024-11-11T20:39:49,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e9f069db477e435f8dbddbbc20fc6499 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e9f069db477e435f8dbddbbc20fc6499 2024-11-11T20:39:49,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:39:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e9f069db477e435f8dbddbbc20fc6499, entries=250, sequenceid=198, filesize=16.5 K 2024-11-11T20:39:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:49,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5b127794e9084765af20f94619937661 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5b127794e9084765af20f94619937661 2024-11-11T20:39:49,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5b127794e9084765af20f94619937661, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T20:39:49,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/fb5ce47e0d334e17adbc9ad0ab21d6a0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fb5ce47e0d334e17adbc9ad0ab21d6a0 2024-11-11T20:39:49,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fb5ce47e0d334e17adbc9ad0ab21d6a0, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T20:39:49,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for fedb58c88f25aed82b02e36155b97788 in 984ms, sequenceid=198, compaction requested=false 2024-11-11T20:39:49,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:49,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:39:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:49,658 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-11T20:39:49,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/694d07f8ca8640928355996999d7e4ff is 50, key is test_row_0/A:col10/1731357588564/Put/seqid=0 2024-11-11T20:39:49,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741876_1052 (size=9757) 2024-11-11T20:39:49,684 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/694d07f8ca8640928355996999d7e4ff 2024-11-11T20:39:49,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/c198312083f743859b3bef66b1be403c is 50, key is test_row_0/B:col10/1731357588564/Put/seqid=0 2024-11-11T20:39:49,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741877_1053 (size=9757) 2024-11-11T20:39:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:49,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:49,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357649796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357649798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357649801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357649802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357649804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357649911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357649911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357649921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357649921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:49,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357649921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357650121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357650121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,126 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/c198312083f743859b3bef66b1be403c 2024-11-11T20:39:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357650131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357650132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357650131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/faa70175d3f94960834fdfe46ed786df is 50, key is test_row_0/C:col10/1731357588564/Put/seqid=0 2024-11-11T20:39:50,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741878_1054 (size=9757) 2024-11-11T20:39:50,168 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/faa70175d3f94960834fdfe46ed786df 2024-11-11T20:39:50,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/694d07f8ca8640928355996999d7e4ff as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/694d07f8ca8640928355996999d7e4ff 2024-11-11T20:39:50,190 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/694d07f8ca8640928355996999d7e4ff, entries=100, sequenceid=210, filesize=9.5 K 2024-11-11T20:39:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/c198312083f743859b3bef66b1be403c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c198312083f743859b3bef66b1be403c 2024-11-11T20:39:50,203 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c198312083f743859b3bef66b1be403c, entries=100, sequenceid=210, filesize=9.5 K 2024-11-11T20:39:50,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/faa70175d3f94960834fdfe46ed786df as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/faa70175d3f94960834fdfe46ed786df 2024-11-11T20:39:50,217 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/faa70175d3f94960834fdfe46ed786df, entries=100, sequenceid=210, filesize=9.5 K 2024-11-11T20:39:50,218 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for fedb58c88f25aed82b02e36155b97788 in 560ms, sequenceid=210, compaction requested=true 2024-11-11T20:39:50,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:50,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:50,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T20:39:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T20:39:50,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T20:39:50,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9630 sec 2024-11-11T20:39:50,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.9760 sec 2024-11-11T20:39:50,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:39:50,401 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:39:50,401 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:39:50,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T20:39:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:39:50,404 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:39:50,405 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:39:50,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:39:50,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:50,434 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:50,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-11T20:39:50,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/434b22ac58504db7a3a2e19f0074547f is 50, key is test_row_0/A:col10/1731357589802/Put/seqid=0 2024-11-11T20:39:50,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357650448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741879_1055 (size=14541) 2024-11-11T20:39:50,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357650448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357650450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357650451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/434b22ac58504db7a3a2e19f0074547f 2024-11-11T20:39:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357650461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5deddefebbf84c999de98589968ca469 is 50, key is test_row_0/B:col10/1731357589802/Put/seqid=0 2024-11-11T20:39:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:39:50,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741880_1056 (size=12151) 2024-11-11T20:39:50,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T20:39:50,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:50,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:50,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:50,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:50,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:50,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:50,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357650562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357650562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357650564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T20:39:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:50,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:39:50,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357650770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357650771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357650771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T20:39:50,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:50,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:50,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:50,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:50,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:50,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5deddefebbf84c999de98589968ca469 2024-11-11T20:39:50,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/198ff39dc9654fb98ece87bf02de42c0 is 50, key is test_row_0/C:col10/1731357589802/Put/seqid=0 2024-11-11T20:39:50,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357650961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741881_1057 (size=12151) 2024-11-11T20:39:50,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/198ff39dc9654fb98ece87bf02de42c0 2024-11-11T20:39:50,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357650973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:50,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/434b22ac58504db7a3a2e19f0074547f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/434b22ac58504db7a3a2e19f0074547f 2024-11-11T20:39:51,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/434b22ac58504db7a3a2e19f0074547f, entries=200, sequenceid=238, filesize=14.2 K 2024-11-11T20:39:51,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5deddefebbf84c999de98589968ca469 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5deddefebbf84c999de98589968ca469 2024-11-11T20:39:51,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5deddefebbf84c999de98589968ca469, entries=150, sequenceid=238, filesize=11.9 K 2024-11-11T20:39:51,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/198ff39dc9654fb98ece87bf02de42c0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/198ff39dc9654fb98ece87bf02de42c0 2024-11-11T20:39:51,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T20:39:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:51,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:51,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/198ff39dc9654fb98ece87bf02de42c0, entries=150, sequenceid=238, filesize=11.9 K 2024-11-11T20:39:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:51,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for fedb58c88f25aed82b02e36155b97788 in 597ms, sequenceid=238, compaction requested=true 2024-11-11T20:39:51,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:51,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:39:51,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:51,031 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:51,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:39:51,031 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:51,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:51,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:39:51,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:39:51,036 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46586 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:51,036 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53756 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:51,037 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/B is initiating minor compaction (all files) 2024-11-11T20:39:51,037 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/A is initiating minor compaction (all files) 2024-11-11T20:39:51,037 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/A in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:51,037 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/B in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:51,037 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/1cd729b47619419689899242b21aa003, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5b127794e9084765af20f94619937661, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c198312083f743859b3bef66b1be403c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5deddefebbf84c999de98589968ca469] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=45.5 K 2024-11-11T20:39:51,037 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/def87914f09e4b99b7dd618cc562ee2f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e9f069db477e435f8dbddbbc20fc6499, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/694d07f8ca8640928355996999d7e4ff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/434b22ac58504db7a3a2e19f0074547f] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=52.5 K 2024-11-11T20:39:51,040 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting def87914f09e4b99b7dd618cc562ee2f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731357588046 2024-11-11T20:39:51,040 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cd729b47619419689899242b21aa003, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731357588046 2024-11-11T20:39:51,041 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9f069db477e435f8dbddbbc20fc6499, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731357588423 2024-11-11T20:39:51,041 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b127794e9084765af20f94619937661, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731357588423 2024-11-11T20:39:51,042 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c198312083f743859b3bef66b1be403c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731357588564 2024-11-11T20:39:51,042 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 694d07f8ca8640928355996999d7e4ff, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731357588564 2024-11-11T20:39:51,044 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5deddefebbf84c999de98589968ca469, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731357589796 2024-11-11T20:39:51,044 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 434b22ac58504db7a3a2e19f0074547f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731357589782 2024-11-11T20:39:51,066 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:51,067 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/60c5590b8d7d43a283f0b30968005c0b is 50, key is test_row_0/B:col10/1731357589802/Put/seqid=0 2024-11-11T20:39:51,077 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:51,078 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/99228532e68442f782fc1c080727789c is 50, key is test_row_0/A:col10/1731357589802/Put/seqid=0 2024-11-11T20:39:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:51,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:51,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:39:51,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e924d318fcfe4f93a06f53990540f1d5 is 50, key is test_row_0/A:col10/1731357591087/Put/seqid=0 2024-11-11T20:39:51,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741882_1058 (size=12663) 2024-11-11T20:39:51,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741883_1059 (size=12663) 2024-11-11T20:39:51,125 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/60c5590b8d7d43a283f0b30968005c0b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/60c5590b8d7d43a283f0b30968005c0b 2024-11-11T20:39:51,135 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/B of fedb58c88f25aed82b02e36155b97788 into 60c5590b8d7d43a283f0b30968005c0b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:51,135 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:51,135 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/B, priority=12, startTime=1731357591031; duration=0sec 2024-11-11T20:39:51,136 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:51,136 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:B 2024-11-11T20:39:51,136 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:51,139 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46586 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:51,139 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/C is initiating minor compaction (all files) 2024-11-11T20:39:51,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741884_1060 (size=14541) 2024-11-11T20:39:51,140 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/C in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:51,140 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/feb037f937034510bc0cc78c599443a6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fb5ce47e0d334e17adbc9ad0ab21d6a0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/faa70175d3f94960834fdfe46ed786df, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/198ff39dc9654fb98ece87bf02de42c0] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=45.5 K 2024-11-11T20:39:51,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e924d318fcfe4f93a06f53990540f1d5 2024-11-11T20:39:51,142 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting feb037f937034510bc0cc78c599443a6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731357588046 2024-11-11T20:39:51,143 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fb5ce47e0d334e17adbc9ad0ab21d6a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731357588423 2024-11-11T20:39:51,145 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting faa70175d3f94960834fdfe46ed786df, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731357588564 2024-11-11T20:39:51,146 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 198ff39dc9654fb98ece87bf02de42c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731357589796 2024-11-11T20:39:51,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/7a28b6e8039248418be7e4b39e0e3e2a is 50, key is test_row_0/B:col10/1731357591087/Put/seqid=0 2024-11-11T20:39:51,179 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#C#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:51,179 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/72d0397e923548a397e0e82d767d8bd7 is 50, key is test_row_0/C:col10/1731357589802/Put/seqid=0 2024-11-11T20:39:51,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T20:39:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:51,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:51,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741885_1061 (size=12151) 2024-11-11T20:39:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:51,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/7a28b6e8039248418be7e4b39e0e3e2a 2024-11-11T20:39:51,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357651195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357651195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357651209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741886_1062 (size=12663) 2024-11-11T20:39:51,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/4f73d5b3fa184e16a667a82137c75205 is 50, key is test_row_0/C:col10/1731357591087/Put/seqid=0 2024-11-11T20:39:51,234 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/72d0397e923548a397e0e82d767d8bd7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/72d0397e923548a397e0e82d767d8bd7 2024-11-11T20:39:51,247 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/C of fedb58c88f25aed82b02e36155b97788 into 72d0397e923548a397e0e82d767d8bd7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:51,247 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:51,250 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/C, priority=12, startTime=1731357591031; duration=0sec 2024-11-11T20:39:51,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:51,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:C 2024-11-11T20:39:51,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741887_1063 (size=12151) 2024-11-11T20:39:51,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/4f73d5b3fa184e16a667a82137c75205 2024-11-11T20:39:51,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e924d318fcfe4f93a06f53990540f1d5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e924d318fcfe4f93a06f53990540f1d5 2024-11-11T20:39:51,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e924d318fcfe4f93a06f53990540f1d5, entries=200, sequenceid=249, filesize=14.2 K 2024-11-11T20:39:51,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/7a28b6e8039248418be7e4b39e0e3e2a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/7a28b6e8039248418be7e4b39e0e3e2a 2024-11-11T20:39:51,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/7a28b6e8039248418be7e4b39e0e3e2a, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T20:39:51,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/4f73d5b3fa184e16a667a82137c75205 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4f73d5b3fa184e16a667a82137c75205 2024-11-11T20:39:51,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357651321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357651321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357651321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4f73d5b3fa184e16a667a82137c75205, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T20:39:51,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for fedb58c88f25aed82b02e36155b97788 in 249ms, sequenceid=249, compaction requested=false 2024-11-11T20:39:51,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:51,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T20:39:51,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:51,348 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T20:39:51,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/bec49577a7ff450586822797869e8eb2 is 50, key is test_row_0/A:col10/1731357591186/Put/seqid=0 2024-11-11T20:39:51,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741888_1064 (size=12301) 2024-11-11T20:39:51,533 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/99228532e68442f782fc1c080727789c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/99228532e68442f782fc1c080727789c 2024-11-11T20:39:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:51,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:39:51,544 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/A of fedb58c88f25aed82b02e36155b97788 into 99228532e68442f782fc1c080727789c(size=12.4 K), total size for store is 26.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:51,544 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:51,544 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/A, priority=12, startTime=1731357591031; duration=0sec 2024-11-11T20:39:51,545 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:51,545 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:A 2024-11-11T20:39:51,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357651555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357651556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357651556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357651671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357651671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357651671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,782 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/bec49577a7ff450586822797869e8eb2 2024-11-11T20:39:51,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/30bdd078074d482f86d03413c16200fe is 50, key is test_row_0/B:col10/1731357591186/Put/seqid=0 2024-11-11T20:39:51,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741889_1065 (size=12301) 2024-11-11T20:39:51,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357651881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357651882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357651882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357651980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,986 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in fedb58c88f25aed82b02e36155b97788 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:39:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357651991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:51,995 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in fedb58c88f25aed82b02e36155b97788 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:39:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:52,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357652191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:52,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357652191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:52,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357652191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:52,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:52,205 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/30bdd078074d482f86d03413c16200fe 2024-11-11T20:39:52,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/8dc92b8f924c466ea84c8ad4b2d4c147 is 50, key is test_row_0/C:col10/1731357591186/Put/seqid=0 2024-11-11T20:39:52,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741890_1066 (size=12301) 2024-11-11T20:39:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:39:52,644 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/8dc92b8f924c466ea84c8ad4b2d4c147 2024-11-11T20:39:52,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/bec49577a7ff450586822797869e8eb2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/bec49577a7ff450586822797869e8eb2 2024-11-11T20:39:52,671 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/bec49577a7ff450586822797869e8eb2, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T20:39:52,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/30bdd078074d482f86d03413c16200fe as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/30bdd078074d482f86d03413c16200fe 2024-11-11T20:39:52,680 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/30bdd078074d482f86d03413c16200fe, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T20:39:52,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/8dc92b8f924c466ea84c8ad4b2d4c147 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/8dc92b8f924c466ea84c8ad4b2d4c147 2024-11-11T20:39:52,692 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/8dc92b8f924c466ea84c8ad4b2d4c147, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T20:39:52,694 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for fedb58c88f25aed82b02e36155b97788 in 1346ms, sequenceid=278, compaction requested=true 2024-11-11T20:39:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T20:39:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T20:39:52,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T20:39:52,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2910 sec 2024-11-11T20:39:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:52,706 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:52,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:39:52,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.3070 sec 2024-11-11T20:39:52,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e820b5da29b64868ab6b9d0fe5749f06 is 50, key is test_row_0/A:col10/1731357591554/Put/seqid=0 2024-11-11T20:39:52,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741891_1067 (size=14737) 2024-11-11T20:39:52,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e820b5da29b64868ab6b9d0fe5749f06 2024-11-11T20:39:52,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/65db89395e9b429cafb90a28ad77d34f is 50, key is test_row_0/B:col10/1731357591554/Put/seqid=0 2024-11-11T20:39:52,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357652781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:52,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:52,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357652786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357652783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:52,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741892_1068 (size=9857) 2024-11-11T20:39:52,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357652891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357652901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357652901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357653101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357653111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357653111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/65db89395e9b429cafb90a28ad77d34f 2024-11-11T20:39:53,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/953a2aac61b54609b690680ee6ae7032 is 50, key is test_row_0/C:col10/1731357591554/Put/seqid=0 2024-11-11T20:39:53,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741893_1069 (size=9857) 2024-11-11T20:39:53,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/953a2aac61b54609b690680ee6ae7032 2024-11-11T20:39:53,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e820b5da29b64868ab6b9d0fe5749f06 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e820b5da29b64868ab6b9d0fe5749f06 2024-11-11T20:39:53,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e820b5da29b64868ab6b9d0fe5749f06, entries=200, sequenceid=290, filesize=14.4 K 2024-11-11T20:39:53,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/65db89395e9b429cafb90a28ad77d34f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/65db89395e9b429cafb90a28ad77d34f 2024-11-11T20:39:53,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/65db89395e9b429cafb90a28ad77d34f, entries=100, sequenceid=290, filesize=9.6 K 2024-11-11T20:39:53,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/953a2aac61b54609b690680ee6ae7032 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/953a2aac61b54609b690680ee6ae7032 2024-11-11T20:39:53,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/953a2aac61b54609b690680ee6ae7032, entries=100, sequenceid=290, filesize=9.6 K 2024-11-11T20:39:53,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fedb58c88f25aed82b02e36155b97788 in 545ms, sequenceid=290, compaction requested=true 2024-11-11T20:39:53,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:53,251 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:53,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:39:53,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:53,252 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:53,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:39:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:39:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:53,255 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:53,255 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/A is initiating minor compaction (all files) 2024-11-11T20:39:53,255 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/A in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:53,255 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/99228532e68442f782fc1c080727789c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e924d318fcfe4f93a06f53990540f1d5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/bec49577a7ff450586822797869e8eb2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e820b5da29b64868ab6b9d0fe5749f06] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=53.0 K 2024-11-11T20:39:53,256 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:53,256 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/B is initiating minor compaction (all files) 2024-11-11T20:39:53,256 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/B in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:53,257 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/60c5590b8d7d43a283f0b30968005c0b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/7a28b6e8039248418be7e4b39e0e3e2a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/30bdd078074d482f86d03413c16200fe, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/65db89395e9b429cafb90a28ad77d34f] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=45.9 K 2024-11-11T20:39:53,257 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99228532e68442f782fc1c080727789c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731357589796 2024-11-11T20:39:53,257 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 60c5590b8d7d43a283f0b30968005c0b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731357589796 2024-11-11T20:39:53,258 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a28b6e8039248418be7e4b39e0e3e2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731357591085 2024-11-11T20:39:53,258 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e924d318fcfe4f93a06f53990540f1d5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731357590446 2024-11-11T20:39:53,259 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 30bdd078074d482f86d03413c16200fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731357591169 2024-11-11T20:39:53,259 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting bec49577a7ff450586822797869e8eb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731357591169 2024-11-11T20:39:53,260 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e820b5da29b64868ab6b9d0fe5749f06, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731357591547 2024-11-11T20:39:53,260 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 65db89395e9b429cafb90a28ad77d34f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731357591554 2024-11-11T20:39:53,276 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:53,276 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/d9f4c2a9577a4836a71b767659d05426 is 50, key is test_row_0/A:col10/1731357591554/Put/seqid=0 2024-11-11T20:39:53,283 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:53,283 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/8923affb5aaa4cbb87d6fa2a7163ea3e is 50, key is test_row_0/B:col10/1731357591554/Put/seqid=0 2024-11-11T20:39:53,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741894_1070 (size=12949) 2024-11-11T20:39:53,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741895_1071 (size=12949) 2024-11-11T20:39:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:53,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:53,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T20:39:53,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/b4c43d2480704365b388eb39a0d4530e is 50, key is test_row_0/A:col10/1731357592778/Put/seqid=0 2024-11-11T20:39:53,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357653435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357653440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357653441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741896_1072 (size=14741) 2024-11-11T20:39:53,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357653551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357653551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357653551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,705 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/d9f4c2a9577a4836a71b767659d05426 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d9f4c2a9577a4836a71b767659d05426 2024-11-11T20:39:53,715 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/8923affb5aaa4cbb87d6fa2a7163ea3e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8923affb5aaa4cbb87d6fa2a7163ea3e 2024-11-11T20:39:53,720 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/A of fedb58c88f25aed82b02e36155b97788 into d9f4c2a9577a4836a71b767659d05426(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:53,721 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:53,721 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/A, priority=12, startTime=1731357593251; duration=0sec 2024-11-11T20:39:53,721 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:53,721 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:A 2024-11-11T20:39:53,721 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:53,724 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:53,724 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/C is initiating minor compaction (all files) 2024-11-11T20:39:53,724 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/C in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:53,724 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/72d0397e923548a397e0e82d767d8bd7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4f73d5b3fa184e16a667a82137c75205, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/8dc92b8f924c466ea84c8ad4b2d4c147, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/953a2aac61b54609b690680ee6ae7032] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=45.9 K 2024-11-11T20:39:53,726 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72d0397e923548a397e0e82d767d8bd7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731357589796 2024-11-11T20:39:53,726 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f73d5b3fa184e16a667a82137c75205, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731357591085 2024-11-11T20:39:53,727 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dc92b8f924c466ea84c8ad4b2d4c147, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731357591169 2024-11-11T20:39:53,728 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 953a2aac61b54609b690680ee6ae7032, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731357591554 2024-11-11T20:39:53,728 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/B of fedb58c88f25aed82b02e36155b97788 into 8923affb5aaa4cbb87d6fa2a7163ea3e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:53,728 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:53,728 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/B, priority=12, startTime=1731357593252; duration=0sec 2024-11-11T20:39:53,728 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:53,728 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:B 2024-11-11T20:39:53,745 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#C#compaction#60 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:53,746 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/e43c08fed4f74d85ad3783959189a25b is 50, key is test_row_0/C:col10/1731357591554/Put/seqid=0 2024-11-11T20:39:53,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357653761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357653761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357653761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:53,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741897_1073 (size=12949) 2024-11-11T20:39:53,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/b4c43d2480704365b388eb39a0d4530e 2024-11-11T20:39:53,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/37b28015a36e4395a49cd231fde3042b is 50, key is test_row_0/B:col10/1731357592778/Put/seqid=0 2024-11-11T20:39:53,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741898_1074 (size=12301) 2024-11-11T20:39:53,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/37b28015a36e4395a49cd231fde3042b 2024-11-11T20:39:53,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/9c6b82c1bd9544daabb3773290fce97c is 50, key is test_row_0/C:col10/1731357592778/Put/seqid=0 2024-11-11T20:39:53,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741899_1075 (size=12301) 2024-11-11T20:39:54,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357654001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,002 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in fedb58c88f25aed82b02e36155b97788 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:39:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357654010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,013 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in fedb58c88f25aed82b02e36155b97788 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:39:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357654071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357654071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357654071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,179 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/e43c08fed4f74d85ad3783959189a25b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/e43c08fed4f74d85ad3783959189a25b 2024-11-11T20:39:54,192 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/C of fedb58c88f25aed82b02e36155b97788 into e43c08fed4f74d85ad3783959189a25b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:54,192 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:54,192 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/C, priority=12, startTime=1731357593253; duration=0sec 2024-11-11T20:39:54,192 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:54,192 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:C 2024-11-11T20:39:54,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/9c6b82c1bd9544daabb3773290fce97c 2024-11-11T20:39:54,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/b4c43d2480704365b388eb39a0d4530e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b4c43d2480704365b388eb39a0d4530e 2024-11-11T20:39:54,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b4c43d2480704365b388eb39a0d4530e, entries=200, sequenceid=316, filesize=14.4 K 2024-11-11T20:39:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/37b28015a36e4395a49cd231fde3042b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/37b28015a36e4395a49cd231fde3042b 2024-11-11T20:39:54,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/37b28015a36e4395a49cd231fde3042b, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T20:39:54,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/9c6b82c1bd9544daabb3773290fce97c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/9c6b82c1bd9544daabb3773290fce97c 2024-11-11T20:39:54,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/9c6b82c1bd9544daabb3773290fce97c, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T20:39:54,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fedb58c88f25aed82b02e36155b97788 in 923ms, sequenceid=316, compaction requested=false 2024-11-11T20:39:54,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:39:54,571 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:39:54,572 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:39:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T20:39:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:39:54,576 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:39:54,580 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:39:54,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:39:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:54,584 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:54,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:39:54,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/fe16f212762541a287bd1b36679d85f0 is 50, key is test_row_0/A:col10/1731357594582/Put/seqid=0 2024-11-11T20:39:54,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741900_1076 (size=14741) 2024-11-11T20:39:54,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/fe16f212762541a287bd1b36679d85f0 2024-11-11T20:39:54,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357654638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5f24bbb3a8b74efcb25edb2792c06d15 is 50, key is test_row_0/B:col10/1731357594582/Put/seqid=0 2024-11-11T20:39:54,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357654641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357654646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:39:54,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741901_1077 (size=12301) 2024-11-11T20:39:54,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5f24bbb3a8b74efcb25edb2792c06d15 2024-11-11T20:39:54,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/96b7f769f04e4d78943e235eba32edbb is 50, key is test_row_0/C:col10/1731357594582/Put/seqid=0 2024-11-11T20:39:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741902_1078 (size=12301) 2024-11-11T20:39:54,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/96b7f769f04e4d78943e235eba32edbb 2024-11-11T20:39:54,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/fe16f212762541a287bd1b36679d85f0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/fe16f212762541a287bd1b36679d85f0 2024-11-11T20:39:54,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:39:54,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:54,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:54,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:54,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:54,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/fe16f212762541a287bd1b36679d85f0, entries=200, sequenceid=331, filesize=14.4 K 2024-11-11T20:39:54,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:54,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/5f24bbb3a8b74efcb25edb2792c06d15 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5f24bbb3a8b74efcb25edb2792c06d15 2024-11-11T20:39:54,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5f24bbb3a8b74efcb25edb2792c06d15, entries=150, sequenceid=331, filesize=12.0 K 2024-11-11T20:39:54,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/96b7f769f04e4d78943e235eba32edbb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/96b7f769f04e4d78943e235eba32edbb 2024-11-11T20:39:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357654751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/96b7f769f04e4d78943e235eba32edbb, entries=150, sequenceid=331, filesize=12.0 K 2024-11-11T20:39:54,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for fedb58c88f25aed82b02e36155b97788 in 176ms, sequenceid=331, compaction requested=true 2024-11-11T20:39:54,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:54,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:39:54,761 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:39:54,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:54,761 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:39:54,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:39:54,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:54,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:39:54,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:54,765 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:39:54,765 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:39:54,765 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/B is initiating minor compaction (all files) 2024-11-11T20:39:54,765 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/A is initiating minor compaction (all files) 2024-11-11T20:39:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:54,765 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/A in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:54,765 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/B in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:54,765 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d9f4c2a9577a4836a71b767659d05426, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b4c43d2480704365b388eb39a0d4530e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/fe16f212762541a287bd1b36679d85f0] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=41.4 K 2024-11-11T20:39:54,765 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8923affb5aaa4cbb87d6fa2a7163ea3e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/37b28015a36e4395a49cd231fde3042b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5f24bbb3a8b74efcb25edb2792c06d15] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=36.7 K 2024-11-11T20:39:54,765 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:54,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T20:39:54,766 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9f4c2a9577a4836a71b767659d05426, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731357591169 2024-11-11T20:39:54,766 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8923affb5aaa4cbb87d6fa2a7163ea3e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731357591169 2024-11-11T20:39:54,767 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c43d2480704365b388eb39a0d4530e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731357592743 2024-11-11T20:39:54,767 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b28015a36e4395a49cd231fde3042b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731357592743 2024-11-11T20:39:54,767 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe16f212762541a287bd1b36679d85f0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731357593430 2024-11-11T20:39:54,767 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f24bbb3a8b74efcb25edb2792c06d15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731357593434 2024-11-11T20:39:54,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/abc4067db9ef4f32a2c0bfbbdab0ab57 is 50, key is test_row_0/A:col10/1731357594763/Put/seqid=0 2024-11-11T20:39:54,783 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:54,784 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/913a577098014de0aac00d341a8bc0bc is 50, key is test_row_0/A:col10/1731357594582/Put/seqid=0 2024-11-11T20:39:54,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357654787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357654792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,803 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#B#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:54,804 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/ceb91d19f5b642058e42cf15134c2d7b is 50, key is test_row_0/B:col10/1731357594582/Put/seqid=0 2024-11-11T20:39:54,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741904_1080 (size=13051) 2024-11-11T20:39:54,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741903_1079 (size=12301) 2024-11-11T20:39:54,833 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/913a577098014de0aac00d341a8bc0bc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/913a577098014de0aac00d341a8bc0bc 2024-11-11T20:39:54,847 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fedb58c88f25aed82b02e36155b97788/A of fedb58c88f25aed82b02e36155b97788 into 913a577098014de0aac00d341a8bc0bc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:54,847 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:54,847 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/A, priority=13, startTime=1731357594761; duration=0sec 2024-11-11T20:39:54,847 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:54,848 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:A 2024-11-11T20:39:54,848 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:39:54,850 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:39:54,851 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/C is initiating minor compaction (all files) 2024-11-11T20:39:54,851 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/C in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:54,852 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/e43c08fed4f74d85ad3783959189a25b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/9c6b82c1bd9544daabb3773290fce97c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/96b7f769f04e4d78943e235eba32edbb] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=36.7 K 2024-11-11T20:39:54,852 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e43c08fed4f74d85ad3783959189a25b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731357591169 2024-11-11T20:39:54,853 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c6b82c1bd9544daabb3773290fce97c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731357592743 2024-11-11T20:39:54,853 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b7f769f04e4d78943e235eba32edbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731357593434 2024-11-11T20:39:54,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741905_1081 (size=13051) 2024-11-11T20:39:54,869 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/ceb91d19f5b642058e42cf15134c2d7b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ceb91d19f5b642058e42cf15134c2d7b 2024-11-11T20:39:54,872 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#C#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:54,872 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/10d6fa6eb6784c48a2433dd6fbc05d61 is 50, key is test_row_0/C:col10/1731357594582/Put/seqid=0 2024-11-11T20:39:54,882 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fedb58c88f25aed82b02e36155b97788/B of fedb58c88f25aed82b02e36155b97788 into ceb91d19f5b642058e42cf15134c2d7b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:54,883 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:54,883 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/B, priority=13, startTime=1731357594761; duration=0sec 2024-11-11T20:39:54,883 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:54,883 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:B 2024-11-11T20:39:54,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:39:54,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:54,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:54,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:54,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:54,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:39:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:54,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357654901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357654901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:54,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741906_1082 (size=13051) 2024-11-11T20:39:54,936 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/10d6fa6eb6784c48a2433dd6fbc05d61 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/10d6fa6eb6784c48a2433dd6fbc05d61 2024-11-11T20:39:54,945 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fedb58c88f25aed82b02e36155b97788/C of fedb58c88f25aed82b02e36155b97788 into 10d6fa6eb6784c48a2433dd6fbc05d61(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:54,945 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:54,946 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/C, priority=13, startTime=1731357594761; duration=0sec 2024-11-11T20:39:54,946 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:54,946 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:C 2024-11-11T20:39:54,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357654960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:55,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:39:55,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:55,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357655111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:55,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:55,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357655111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:39:55,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:39:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/abc4067db9ef4f32a2c0bfbbdab0ab57 2024-11-11T20:39:55,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357655270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:55,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/b4f36ec5a72f4a53b6933423b8e717a4 is 50, key is test_row_0/B:col10/1731357594763/Put/seqid=0 2024-11-11T20:39:55,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741907_1083 (size=12301) 2024-11-11T20:39:55,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/b4f36ec5a72f4a53b6933423b8e717a4 2024-11-11T20:39:55,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:39:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/2399bed7bc10424f8e8aba8e841451ca is 50, key is test_row_0/C:col10/1731357594763/Put/seqid=0 2024-11-11T20:39:55,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741908_1084 (size=12301) 2024-11-11T20:39:55,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/2399bed7bc10424f8e8aba8e841451ca 2024-11-11T20:39:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/abc4067db9ef4f32a2c0bfbbdab0ab57 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/abc4067db9ef4f32a2c0bfbbdab0ab57 2024-11-11T20:39:55,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/abc4067db9ef4f32a2c0bfbbdab0ab57, entries=150, sequenceid=358, filesize=12.0 K 2024-11-11T20:39:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/b4f36ec5a72f4a53b6933423b8e717a4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/b4f36ec5a72f4a53b6933423b8e717a4 2024-11-11T20:39:55,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357655421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:55,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357655421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:55,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/b4f36ec5a72f4a53b6933423b8e717a4, entries=150, sequenceid=358, filesize=12.0 K 2024-11-11T20:39:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:55,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/2399bed7bc10424f8e8aba8e841451ca as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2399bed7bc10424f8e8aba8e841451ca 2024-11-11T20:39:55,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2399bed7bc10424f8e8aba8e841451ca, entries=150, sequenceid=358, filesize=12.0 K 2024-11-11T20:39:55,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for fedb58c88f25aed82b02e36155b97788 in 690ms, sequenceid=358, compaction requested=false 2024-11-11T20:39:55,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:55,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:39:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,514 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-11T20:39:55,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/dfef0f15985d460ea526526efcd745cc is 50, key is test_row_0/A:col10/1731357594789/Put/seqid=0 2024-11-11T20:39:55,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741909_1085 (size=12301) 2024-11-11T20:39:55,533 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/dfef0f15985d460ea526526efcd745cc 2024-11-11T20:39:55,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/57e593bf0c394f149750f7fdc0b97147 is 50, key is test_row_0/B:col10/1731357594789/Put/seqid=0 2024-11-11T20:39:55,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741910_1086 (size=12301) 2024-11-11T20:39:55,576 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/57e593bf0c394f149750f7fdc0b97147 2024-11-11T20:39:55,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/f3eb86ee331248f49875e7b442897778 is 50, key is test_row_0/C:col10/1731357594789/Put/seqid=0 2024-11-11T20:39:55,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741911_1087 (size=12301) 2024-11-11T20:39:55,618 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/f3eb86ee331248f49875e7b442897778 2024-11-11T20:39:55,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/dfef0f15985d460ea526526efcd745cc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dfef0f15985d460ea526526efcd745cc 2024-11-11T20:39:55,647 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dfef0f15985d460ea526526efcd745cc, entries=150, sequenceid=370, filesize=12.0 K 2024-11-11T20:39:55,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/57e593bf0c394f149750f7fdc0b97147 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/57e593bf0c394f149750f7fdc0b97147 2024-11-11T20:39:55,657 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/57e593bf0c394f149750f7fdc0b97147, entries=150, sequenceid=370, filesize=12.0 K 2024-11-11T20:39:55,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/f3eb86ee331248f49875e7b442897778 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f3eb86ee331248f49875e7b442897778 2024-11-11T20:39:55,668 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f3eb86ee331248f49875e7b442897778, entries=150, sequenceid=370, filesize=12.0 K 2024-11-11T20:39:55,670 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for fedb58c88f25aed82b02e36155b97788 in 156ms, sequenceid=370, compaction requested=true 2024-11-11T20:39:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T20:39:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T20:39:55,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T20:39:55,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0930 sec 2024-11-11T20:39:55,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1060 sec 2024-11-11T20:39:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:39:55,711 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:39:55,712 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:39:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T20:39:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:39:55,715 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:39:55,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:39:55,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:39:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:39:55,823 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:55,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:39:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:55,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/7f87deabc38c4ac7bfc178049b79d3cb is 50, key is test_row_0/A:col10/1731357595811/Put/seqid=0 2024-11-11T20:39:55,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741912_1088 (size=12301) 2024-11-11T20:39:55,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/7f87deabc38c4ac7bfc178049b79d3cb 2024-11-11T20:39:55,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T20:39:55,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:55,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:55,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:55,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/f74b0d2b54104ce98ec5e30b3a10c2a4 is 50, key is test_row_0/B:col10/1731357595811/Put/seqid=0 2024-11-11T20:39:55,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741913_1089 (size=12301) 2024-11-11T20:39:55,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/f74b0d2b54104ce98ec5e30b3a10c2a4 2024-11-11T20:39:55,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357655956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:55,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:55,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/fba1518798ef4a578341923e6a7dd371 is 50, key is test_row_0/C:col10/1731357595811/Put/seqid=0 2024-11-11T20:39:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357655965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357655967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741914_1090 (size=12301) 2024-11-11T20:39:56,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/fba1518798ef4a578341923e6a7dd371 2024-11-11T20:39:56,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/7f87deabc38c4ac7bfc178049b79d3cb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/7f87deabc38c4ac7bfc178049b79d3cb 2024-11-11T20:39:56,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/7f87deabc38c4ac7bfc178049b79d3cb, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T20:39:56,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/f74b0d2b54104ce98ec5e30b3a10c2a4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/f74b0d2b54104ce98ec5e30b3a10c2a4 2024-11-11T20:39:56,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/f74b0d2b54104ce98ec5e30b3a10c2a4, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T20:39:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:39:56,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/fba1518798ef4a578341923e6a7dd371 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fba1518798ef4a578341923e6a7dd371 2024-11-11T20:39:56,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T20:39:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:56,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fba1518798ef4a578341923e6a7dd371, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T20:39:56,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fedb58c88f25aed82b02e36155b97788 in 229ms, sequenceid=381, compaction requested=true 2024-11-11T20:39:56,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:56,053 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:56,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:39:56,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:56,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:39:56,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:56,053 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:56,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:39:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:56,055 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:56,055 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/A is initiating minor compaction (all files) 2024-11-11T20:39:56,055 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/A in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,055 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/913a577098014de0aac00d341a8bc0bc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/abc4067db9ef4f32a2c0bfbbdab0ab57, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dfef0f15985d460ea526526efcd745cc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/7f87deabc38c4ac7bfc178049b79d3cb] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=48.8 K 2024-11-11T20:39:56,055 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:56,055 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/B is initiating minor compaction (all files) 2024-11-11T20:39:56,055 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/B in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,056 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ceb91d19f5b642058e42cf15134c2d7b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/b4f36ec5a72f4a53b6933423b8e717a4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/57e593bf0c394f149750f7fdc0b97147, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/f74b0d2b54104ce98ec5e30b3a10c2a4] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=48.8 K 2024-11-11T20:39:56,056 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 913a577098014de0aac00d341a8bc0bc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731357593434 2024-11-11T20:39:56,057 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb91d19f5b642058e42cf15134c2d7b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731357593434 2024-11-11T20:39:56,058 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting abc4067db9ef4f32a2c0bfbbdab0ab57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731357594640 2024-11-11T20:39:56,058 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b4f36ec5a72f4a53b6933423b8e717a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731357594640 2024-11-11T20:39:56,058 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfef0f15985d460ea526526efcd745cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731357594778 2024-11-11T20:39:56,059 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e593bf0c394f149750f7fdc0b97147, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731357594778 2024-11-11T20:39:56,059 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f87deabc38c4ac7bfc178049b79d3cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731357595808 2024-11-11T20:39:56,060 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f74b0d2b54104ce98ec5e30b3a10c2a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731357595808 2024-11-11T20:39:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:56,082 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:56,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T20:39:56,088 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:56,089 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/ce4828381e504504b94b262df668f53c is 50, key is test_row_0/A:col10/1731357595811/Put/seqid=0 2024-11-11T20:39:56,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/35e7953056654d928599e225ddbbbc2d is 50, key is test_row_0/A:col10/1731357595962/Put/seqid=0 2024-11-11T20:39:56,094 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#B#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:56,095 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/c329de2d8ddf4fab91df6a524ca3dabe is 50, key is test_row_0/B:col10/1731357595811/Put/seqid=0 2024-11-11T20:39:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357656099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357656104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357656102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741915_1091 (size=13187) 2024-11-11T20:39:56,128 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/ce4828381e504504b94b262df668f53c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ce4828381e504504b94b262df668f53c 2024-11-11T20:39:56,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741916_1092 (size=14741) 2024-11-11T20:39:56,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/35e7953056654d928599e225ddbbbc2d 2024-11-11T20:39:56,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741917_1093 (size=13187) 2024-11-11T20:39:56,145 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/A of fedb58c88f25aed82b02e36155b97788 into ce4828381e504504b94b262df668f53c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:56,145 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:56,146 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/A, priority=12, startTime=1731357596053; duration=0sec 2024-11-11T20:39:56,146 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:56,146 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:A 2024-11-11T20:39:56,146 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:56,151 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:56,151 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/C is initiating minor compaction (all files) 2024-11-11T20:39:56,151 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/C in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,151 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/10d6fa6eb6784c48a2433dd6fbc05d61, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2399bed7bc10424f8e8aba8e841451ca, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f3eb86ee331248f49875e7b442897778, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fba1518798ef4a578341923e6a7dd371] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=48.8 K 2024-11-11T20:39:56,152 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10d6fa6eb6784c48a2433dd6fbc05d61, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731357593434 2024-11-11T20:39:56,153 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2399bed7bc10424f8e8aba8e841451ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731357594640 2024-11-11T20:39:56,153 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3eb86ee331248f49875e7b442897778, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731357594778 2024-11-11T20:39:56,154 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba1518798ef4a578341923e6a7dd371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731357595808 2024-11-11T20:39:56,156 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/c329de2d8ddf4fab91df6a524ca3dabe as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c329de2d8ddf4fab91df6a524ca3dabe 2024-11-11T20:39:56,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/a8a6ca01e39c48dc98b581ba23d1231f is 50, key is test_row_0/B:col10/1731357595962/Put/seqid=0 2024-11-11T20:39:56,171 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/B of fedb58c88f25aed82b02e36155b97788 into c329de2d8ddf4fab91df6a524ca3dabe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:56,173 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:56,174 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/B, priority=12, startTime=1731357596053; duration=0sec 2024-11-11T20:39:56,174 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:56,174 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:B 2024-11-11T20:39:56,183 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:56,184 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/92cfccd59c494de4b14b2c786b2dc6a7 is 50, key is test_row_0/C:col10/1731357595811/Put/seqid=0 2024-11-11T20:39:56,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741918_1094 (size=12301) 2024-11-11T20:39:56,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T20:39:56,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:56,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741919_1095 (size=13187) 2024-11-11T20:39:56,204 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/92cfccd59c494de4b14b2c786b2dc6a7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/92cfccd59c494de4b14b2c786b2dc6a7 2024-11-11T20:39:56,212 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/C of fedb58c88f25aed82b02e36155b97788 into 92cfccd59c494de4b14b2c786b2dc6a7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:56,212 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:56,212 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/C, priority=12, startTime=1731357596053; duration=0sec 2024-11-11T20:39:56,212 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:56,212 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:C 2024-11-11T20:39:56,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357656221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357656221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357656221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:39:56,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T20:39:56,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:56,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357656431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357656431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357656431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T20:39:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/a8a6ca01e39c48dc98b581ba23d1231f 2024-11-11T20:39:56,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/5e2dc77eea8c49d69f26b38aecaf4a99 is 50, key is test_row_0/C:col10/1731357595962/Put/seqid=0 2024-11-11T20:39:56,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741920_1096 (size=12301) 2024-11-11T20:39:56,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T20:39:56,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:56,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357656740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357656741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357656741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:56,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:56,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T20:39:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:39:56,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T20:39:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:56,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:57,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/5e2dc77eea8c49d69f26b38aecaf4a99 2024-11-11T20:39:57,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/35e7953056654d928599e225ddbbbc2d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/35e7953056654d928599e225ddbbbc2d 2024-11-11T20:39:57,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/35e7953056654d928599e225ddbbbc2d, entries=200, sequenceid=407, filesize=14.4 K 2024-11-11T20:39:57,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/a8a6ca01e39c48dc98b581ba23d1231f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/a8a6ca01e39c48dc98b581ba23d1231f 2024-11-11T20:39:57,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/a8a6ca01e39c48dc98b581ba23d1231f, entries=150, sequenceid=407, filesize=12.0 K 2024-11-11T20:39:57,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/5e2dc77eea8c49d69f26b38aecaf4a99 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5e2dc77eea8c49d69f26b38aecaf4a99 2024-11-11T20:39:57,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5e2dc77eea8c49d69f26b38aecaf4a99, entries=150, sequenceid=407, filesize=12.0 K 2024-11-11T20:39:57,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fedb58c88f25aed82b02e36155b97788 in 971ms, sequenceid=407, compaction requested=false 2024-11-11T20:39:57,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:57,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T20:39:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:57,131 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T20:39:57,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/f0138c9d40eb4b33bb75eea97a19fc54 is 50, key is test_row_0/A:col10/1731357596101/Put/seqid=0 2024-11-11T20:39:57,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741921_1097 (size=12301) 2024-11-11T20:39:57,163 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/f0138c9d40eb4b33bb75eea97a19fc54 2024-11-11T20:39:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/8e03aba358914d278955875c1abb2165 is 50, key is test_row_0/B:col10/1731357596101/Put/seqid=0 2024-11-11T20:39:57,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741922_1098 (size=12301) 2024-11-11T20:39:57,201 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/8e03aba358914d278955875c1abb2165 2024-11-11T20:39:57,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/b03a6815dac64d2c82cc198cee92d994 is 50, key is test_row_0/C:col10/1731357596101/Put/seqid=0 2024-11-11T20:39:57,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741923_1099 (size=12301) 2024-11-11T20:39:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:57,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:57,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357657317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357657321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357657321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357657431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357657431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357657431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,618 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/b03a6815dac64d2c82cc198cee92d994 2024-11-11T20:39:57,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/f0138c9d40eb4b33bb75eea97a19fc54 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/f0138c9d40eb4b33bb75eea97a19fc54 2024-11-11T20:39:57,634 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/f0138c9d40eb4b33bb75eea97a19fc54, entries=150, sequenceid=420, filesize=12.0 K 2024-11-11T20:39:57,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/8e03aba358914d278955875c1abb2165 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e03aba358914d278955875c1abb2165 2024-11-11T20:39:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357657641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357657641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357657641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,644 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e03aba358914d278955875c1abb2165, entries=150, sequenceid=420, filesize=12.0 K 2024-11-11T20:39:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/b03a6815dac64d2c82cc198cee92d994 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/b03a6815dac64d2c82cc198cee92d994 2024-11-11T20:39:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,654 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/b03a6815dac64d2c82cc198cee92d994, entries=150, sequenceid=420, filesize=12.0 K 2024-11-11T20:39:57,655 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for fedb58c88f25aed82b02e36155b97788 in 524ms, sequenceid=420, compaction requested=true 2024-11-11T20:39:57,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:57,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:57,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T20:39:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T20:39:57,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T20:39:57,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9400 sec 2024-11-11T20:39:57,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.9490 sec 2024-11-11T20:39:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:39:57,864 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:39:57,870 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:39:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T20:39:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T20:39:57,875 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:39:57,884 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:39:57,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:39:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:39:57,957 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:39:57,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T20:39:57,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/4ef13e64e6624ec7a427984d6dea1626 is 50, key is test_row_0/A:col10/1731357597954/Put/seqid=0 2024-11-11T20:39:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T20:39:57,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357657976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357657978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357657984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741924_1100 (size=12301) 2024-11-11T20:39:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48556 deadline: 1731357658021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,029 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in fedb58c88f25aed82b02e36155b97788 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:39:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48528 deadline: 1731357658041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:58,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:58,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,045 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in fedb58c88f25aed82b02e36155b97788 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:39:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357658090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357658101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357658101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T20:39:58,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357658301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357658311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357658311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/4ef13e64e6624ec7a427984d6dea1626 2024-11-11T20:39:58,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/2df44e796f7a467bab823fbb96f8240c is 50, key is test_row_0/B:col10/1731357597954/Put/seqid=0 2024-11-11T20:39:58,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741925_1101 (size=12301) 2024-11-11T20:39:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T20:39:58,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357658611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357658621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357658621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:58,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:58,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:58,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:58,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:58,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/2df44e796f7a467bab823fbb96f8240c 2024-11-11T20:39:58,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/bd4c6b1a228e486dac496cb0fc7f3336 is 50, key is test_row_0/C:col10/1731357597954/Put/seqid=0 2024-11-11T20:39:58,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741926_1102 (size=12301) 2024-11-11T20:39:58,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:58,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T20:39:59,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:59,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:59,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:59,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:59,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:59,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:59,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357659121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:59,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:59,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:59,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:59,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357659132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:59,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:39:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357659132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:39:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:39:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:39:59,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/bd4c6b1a228e486dac496cb0fc7f3336 2024-11-11T20:39:59,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/4ef13e64e6624ec7a427984d6dea1626 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4ef13e64e6624ec7a427984d6dea1626 2024-11-11T20:39:59,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:59,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:59,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:39:59,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:59,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:39:59,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4ef13e64e6624ec7a427984d6dea1626, entries=150, sequenceid=449, filesize=12.0 K 2024-11-11T20:39:59,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/2df44e796f7a467bab823fbb96f8240c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/2df44e796f7a467bab823fbb96f8240c 2024-11-11T20:39:59,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/2df44e796f7a467bab823fbb96f8240c, entries=150, sequenceid=449, filesize=12.0 K 2024-11-11T20:39:59,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/bd4c6b1a228e486dac496cb0fc7f3336 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/bd4c6b1a228e486dac496cb0fc7f3336 2024-11-11T20:39:59,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/bd4c6b1a228e486dac496cb0fc7f3336, entries=150, sequenceid=449, filesize=12.0 K 2024-11-11T20:39:59,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for fedb58c88f25aed82b02e36155b97788 in 1354ms, sequenceid=449, compaction requested=true 2024-11-11T20:39:59,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:59,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:39:59,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:59,312 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:59,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:39:59,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:59,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:39:59,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T20:39:59,312 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:59,315 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:59,316 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/A is initiating minor compaction (all files) 2024-11-11T20:39:59,316 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/A in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:59,316 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ce4828381e504504b94b262df668f53c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/35e7953056654d928599e225ddbbbc2d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/f0138c9d40eb4b33bb75eea97a19fc54, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4ef13e64e6624ec7a427984d6dea1626] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=51.3 K 2024-11-11T20:39:59,316 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:59,316 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/B is initiating minor compaction (all files) 2024-11-11T20:39:59,316 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/B in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:59,316 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c329de2d8ddf4fab91df6a524ca3dabe, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/a8a6ca01e39c48dc98b581ba23d1231f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e03aba358914d278955875c1abb2165, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/2df44e796f7a467bab823fbb96f8240c] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=48.9 K 2024-11-11T20:39:59,319 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c329de2d8ddf4fab91df6a524ca3dabe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731357595808 2024-11-11T20:39:59,319 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce4828381e504504b94b262df668f53c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731357595808 2024-11-11T20:39:59,319 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a8a6ca01e39c48dc98b581ba23d1231f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731357595952 2024-11-11T20:39:59,320 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35e7953056654d928599e225ddbbbc2d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731357595944 2024-11-11T20:39:59,320 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e03aba358914d278955875c1abb2165, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731357596097 2024-11-11T20:39:59,320 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0138c9d40eb4b33bb75eea97a19fc54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731357596097 2024-11-11T20:39:59,321 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2df44e796f7a467bab823fbb96f8240c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731357597953 2024-11-11T20:39:59,321 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef13e64e6624ec7a427984d6dea1626, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731357597953 2024-11-11T20:39:59,340 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:59,341 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/6c50e424368c48f5bdac5f8ebd396008 is 50, key is test_row_0/B:col10/1731357597954/Put/seqid=0 2024-11-11T20:39:59,345 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#A#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:59,346 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/8dc89b739ed845289f674ab463e69f23 is 50, key is test_row_0/A:col10/1731357597954/Put/seqid=0 2024-11-11T20:39:59,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741928_1104 (size=13323) 2024-11-11T20:39:59,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741927_1103 (size=13323) 2024-11-11T20:39:59,384 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/8dc89b739ed845289f674ab463e69f23 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/8dc89b739ed845289f674ab463e69f23 2024-11-11T20:39:59,398 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/A of fedb58c88f25aed82b02e36155b97788 into 8dc89b739ed845289f674ab463e69f23(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:59,398 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:59,398 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/A, priority=12, startTime=1731357599312; duration=0sec 2024-11-11T20:39:59,398 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:39:59,398 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:A 2024-11-11T20:39:59,398 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:39:59,402 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:39:59,402 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/C is initiating minor compaction (all files) 2024-11-11T20:39:59,402 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/C in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:59,402 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/92cfccd59c494de4b14b2c786b2dc6a7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5e2dc77eea8c49d69f26b38aecaf4a99, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/b03a6815dac64d2c82cc198cee92d994, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/bd4c6b1a228e486dac496cb0fc7f3336] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=48.9 K 2024-11-11T20:39:59,404 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92cfccd59c494de4b14b2c786b2dc6a7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731357595808 2024-11-11T20:39:59,405 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e2dc77eea8c49d69f26b38aecaf4a99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731357595952 2024-11-11T20:39:59,405 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b03a6815dac64d2c82cc198cee92d994, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731357596097 2024-11-11T20:39:59,406 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4c6b1a228e486dac496cb0fc7f3336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731357597953 2024-11-11T20:39:59,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T20:39:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:39:59,433 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T20:39:59,434 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#C#compaction#92 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:39:59,435 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/152c0b0464334851a4d6937044a98b68 is 50, key is test_row_0/C:col10/1731357597954/Put/seqid=0 2024-11-11T20:39:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e2f39e77cfb4414c995d343e1318d37b is 50, key is test_row_0/A:col10/1731357597963/Put/seqid=0 2024-11-11T20:39:59,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741929_1105 (size=13323) 2024-11-11T20:39:59,486 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/152c0b0464334851a4d6937044a98b68 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/152c0b0464334851a4d6937044a98b68 2024-11-11T20:39:59,501 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/C of fedb58c88f25aed82b02e36155b97788 into 152c0b0464334851a4d6937044a98b68(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:59,501 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:59,501 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/C, priority=12, startTime=1731357599312; duration=0sec 2024-11-11T20:39:59,502 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:59,502 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:C 2024-11-11T20:39:59,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741930_1106 (size=12301) 2024-11-11T20:39:59,782 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/6c50e424368c48f5bdac5f8ebd396008 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6c50e424368c48f5bdac5f8ebd396008 2024-11-11T20:39:59,788 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fedb58c88f25aed82b02e36155b97788/B of fedb58c88f25aed82b02e36155b97788 into 6c50e424368c48f5bdac5f8ebd396008(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:39:59,788 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:39:59,788 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/B, priority=12, startTime=1731357599312; duration=0sec 2024-11-11T20:39:59,788 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:39:59,788 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:B 2024-11-11T20:39:59,907 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e2f39e77cfb4414c995d343e1318d37b 2024-11-11T20:39:59,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/c33b9dcc790a4f4e9de268256c8d953b is 50, key is test_row_0/B:col10/1731357597963/Put/seqid=0 2024-11-11T20:39:59,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741931_1107 (size=12301) 2024-11-11T20:40:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T20:40:00,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:40:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:40:00,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357660201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357660201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357660201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357660311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357660311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357660311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,353 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/c33b9dcc790a4f4e9de268256c8d953b 2024-11-11T20:40:00,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/5068e0d63e5a404a97d77353f66211d8 is 50, key is test_row_0/C:col10/1731357597963/Put/seqid=0 2024-11-11T20:40:00,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741932_1108 (size=12301) 2024-11-11T20:40:00,394 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/5068e0d63e5a404a97d77353f66211d8 2024-11-11T20:40:00,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/e2f39e77cfb4414c995d343e1318d37b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e2f39e77cfb4414c995d343e1318d37b 2024-11-11T20:40:00,410 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e2f39e77cfb4414c995d343e1318d37b, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T20:40:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/c33b9dcc790a4f4e9de268256c8d953b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c33b9dcc790a4f4e9de268256c8d953b 2024-11-11T20:40:00,423 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c33b9dcc790a4f4e9de268256c8d953b, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T20:40:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/5068e0d63e5a404a97d77353f66211d8 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5068e0d63e5a404a97d77353f66211d8 2024-11-11T20:40:00,433 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5068e0d63e5a404a97d77353f66211d8, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T20:40:00,438 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for fedb58c88f25aed82b02e36155b97788 in 1005ms, sequenceid=457, compaction requested=false 2024-11-11T20:40:00,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:40:00,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:00,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T20:40:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T20:40:00,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T20:40:00,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5560 sec 2024-11-11T20:40:00,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.5730 sec 2024-11-11T20:40:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:40:00,525 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:00,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-11T20:40:00,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357660524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357660527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357660530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/eeb023bc6e084b9cba6dac30e61fe23f is 50, key is test_row_0/A:col10/1731357600522/Put/seqid=0 2024-11-11T20:40:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741933_1109 (size=19621) 2024-11-11T20:40:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357660641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357660641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357660647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357660851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357660851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357660861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:00,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/eeb023bc6e084b9cba6dac30e61fe23f 2024-11-11T20:40:00,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/8e837bf1cfc74d51a03a35eb60ef6977 is 50, key is test_row_0/B:col10/1731357600522/Put/seqid=0 2024-11-11T20:40:00,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741934_1110 (size=12301) 2024-11-11T20:40:01,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357661161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:01,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357661161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:01,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:01,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357661171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:01,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/8e837bf1cfc74d51a03a35eb60ef6977 2024-11-11T20:40:01,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/648d5601dfea4c9abb62af9b8db68634 is 50, key is test_row_0/C:col10/1731357600522/Put/seqid=0 2024-11-11T20:40:01,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741935_1111 (size=12301) 2024-11-11T20:40:01,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357661671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:01,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357661681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:01,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:01,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357661681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:01,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/648d5601dfea4c9abb62af9b8db68634 2024-11-11T20:40:01,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/eeb023bc6e084b9cba6dac30e61fe23f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/eeb023bc6e084b9cba6dac30e61fe23f 2024-11-11T20:40:01,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/eeb023bc6e084b9cba6dac30e61fe23f, entries=300, sequenceid=491, filesize=19.2 K 2024-11-11T20:40:01,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/8e837bf1cfc74d51a03a35eb60ef6977 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e837bf1cfc74d51a03a35eb60ef6977 2024-11-11T20:40:01,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e837bf1cfc74d51a03a35eb60ef6977, entries=150, sequenceid=491, filesize=12.0 K 2024-11-11T20:40:01,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/648d5601dfea4c9abb62af9b8db68634 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/648d5601dfea4c9abb62af9b8db68634 2024-11-11T20:40:01,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/648d5601dfea4c9abb62af9b8db68634, entries=150, sequenceid=491, filesize=12.0 K 2024-11-11T20:40:01,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for fedb58c88f25aed82b02e36155b97788 in 1316ms, sequenceid=491, compaction requested=true 2024-11-11T20:40:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:40:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:01,841 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:01,841 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fedb58c88f25aed82b02e36155b97788:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:01,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:01,843 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:01,843 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:01,843 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/B is initiating minor compaction (all files) 2024-11-11T20:40:01,843 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/A is initiating minor compaction (all files) 2024-11-11T20:40:01,843 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/B in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:01,843 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/A in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:01,843 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/8dc89b739ed845289f674ab463e69f23, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e2f39e77cfb4414c995d343e1318d37b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/eeb023bc6e084b9cba6dac30e61fe23f] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=44.2 K 2024-11-11T20:40:01,843 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6c50e424368c48f5bdac5f8ebd396008, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c33b9dcc790a4f4e9de268256c8d953b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e837bf1cfc74d51a03a35eb60ef6977] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=37.0 K 2024-11-11T20:40:01,844 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c50e424368c48f5bdac5f8ebd396008, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731357597953 2024-11-11T20:40:01,844 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dc89b739ed845289f674ab463e69f23, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731357597953 2024-11-11T20:40:01,844 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c33b9dcc790a4f4e9de268256c8d953b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731357597963 2024-11-11T20:40:01,844 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2f39e77cfb4414c995d343e1318d37b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731357597963 2024-11-11T20:40:01,845 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e837bf1cfc74d51a03a35eb60ef6977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731357600189 2024-11-11T20:40:01,845 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeb023bc6e084b9cba6dac30e61fe23f, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731357600189 2024-11-11T20:40:01,855 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:01,856 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/ce1a10ea4c4b4e44b089c17631a7d6af is 50, key is test_row_0/A:col10/1731357600522/Put/seqid=0 2024-11-11T20:40:01,858 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:01,858 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/d8eb541d74c3472d9fbf74dfda1e49f8 is 50, key is test_row_0/B:col10/1731357600522/Put/seqid=0 2024-11-11T20:40:01,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741936_1112 (size=13425) 2024-11-11T20:40:01,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741937_1113 (size=13425) 2024-11-11T20:40:01,873 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/d8eb541d74c3472d9fbf74dfda1e49f8 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d8eb541d74c3472d9fbf74dfda1e49f8 2024-11-11T20:40:01,878 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/ce1a10ea4c4b4e44b089c17631a7d6af as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ce1a10ea4c4b4e44b089c17631a7d6af 2024-11-11T20:40:01,882 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fedb58c88f25aed82b02e36155b97788/B of fedb58c88f25aed82b02e36155b97788 into d8eb541d74c3472d9fbf74dfda1e49f8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:01,882 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:40:01,882 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/B, priority=13, startTime=1731357601841; duration=0sec 2024-11-11T20:40:01,882 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:01,882 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:B 2024-11-11T20:40:01,882 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:01,889 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:01,889 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fedb58c88f25aed82b02e36155b97788/C is initiating minor compaction (all files) 2024-11-11T20:40:01,890 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fedb58c88f25aed82b02e36155b97788/C in TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:01,890 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/152c0b0464334851a4d6937044a98b68, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5068e0d63e5a404a97d77353f66211d8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/648d5601dfea4c9abb62af9b8db68634] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp, totalSize=37.0 K 2024-11-11T20:40:01,891 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 152c0b0464334851a4d6937044a98b68, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731357597953 2024-11-11T20:40:01,892 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5068e0d63e5a404a97d77353f66211d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731357597963 2024-11-11T20:40:01,892 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fedb58c88f25aed82b02e36155b97788/A of fedb58c88f25aed82b02e36155b97788 into ce1a10ea4c4b4e44b089c17631a7d6af(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:01,892 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:40:01,892 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/A, priority=13, startTime=1731357601841; duration=0sec 2024-11-11T20:40:01,892 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:01,892 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:A 2024-11-11T20:40:01,892 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 648d5601dfea4c9abb62af9b8db68634, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731357600189 2024-11-11T20:40:01,903 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fedb58c88f25aed82b02e36155b97788#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:01,903 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/b05cfd5cb34a4ff2b6ca6010589787ea is 50, key is test_row_0/C:col10/1731357600522/Put/seqid=0 2024-11-11T20:40:01,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741938_1114 (size=13425) 2024-11-11T20:40:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T20:40:02,041 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:02,041 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T20:40:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T20:40:02,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:02,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:02,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T20:40:02,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T20:40:02,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:02,199 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-11T20:40:02,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/4dc26f74b36342a99582387b0ab88057 is 50, key is test_row_0/A:col10/1731357600528/Put/seqid=0 2024-11-11T20:40:02,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741939_1115 (size=7415) 2024-11-11T20:40:02,316 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/b05cfd5cb34a4ff2b6ca6010589787ea as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/b05cfd5cb34a4ff2b6ca6010589787ea 2024-11-11T20:40:02,322 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fedb58c88f25aed82b02e36155b97788/C of fedb58c88f25aed82b02e36155b97788 into b05cfd5cb34a4ff2b6ca6010589787ea(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:02,322 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:40:02,322 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., storeName=fedb58c88f25aed82b02e36155b97788/C, priority=13, startTime=1731357601841; duration=0sec 2024-11-11T20:40:02,322 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:02,322 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fedb58c88f25aed82b02e36155b97788:C 2024-11-11T20:40:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T20:40:02,608 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/4dc26f74b36342a99582387b0ab88057 2024-11-11T20:40:02,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/19c5d0ea7da9416ea2d560bddcf982a8 is 50, key is test_row_0/B:col10/1731357600528/Put/seqid=0 2024-11-11T20:40:02,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741940_1116 (size=7415) 2024-11-11T20:40:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T20:40:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:40:02,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:40:02,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357662725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:02,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357662727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:02,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:02,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357662732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:02,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357662831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:02,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357662841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:02,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357662840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:03,025 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/19c5d0ea7da9416ea2d560bddcf982a8 2024-11-11T20:40:03,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/f19e5c20a22642df8bfb95d5b81b9999 is 50, key is test_row_0/C:col10/1731357600528/Put/seqid=0 2024-11-11T20:40:03,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731357663040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:03,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741941_1117 (size=7415) 2024-11-11T20:40:03,061 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/f19e5c20a22642df8bfb95d5b81b9999 2024-11-11T20:40:03,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357663062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:03,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:03,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357663062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:03,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/4dc26f74b36342a99582387b0ab88057 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4dc26f74b36342a99582387b0ab88057 2024-11-11T20:40:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:03,079 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4dc26f74b36342a99582387b0ab88057, entries=50, sequenceid=497, filesize=7.2 K 2024-11-11T20:40:03,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/19c5d0ea7da9416ea2d560bddcf982a8 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/19c5d0ea7da9416ea2d560bddcf982a8 2024-11-11T20:40:03,096 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/19c5d0ea7da9416ea2d560bddcf982a8, entries=50, sequenceid=497, filesize=7.2 K 2024-11-11T20:40:03,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/f19e5c20a22642df8bfb95d5b81b9999 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f19e5c20a22642df8bfb95d5b81b9999 2024-11-11T20:40:03,109 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f19e5c20a22642df8bfb95d5b81b9999, entries=50, sequenceid=497, filesize=7.2 K 2024-11-11T20:40:03,112 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=194.56 KB/199230 for fedb58c88f25aed82b02e36155b97788 in 912ms, sequenceid=497, compaction requested=false 2024-11-11T20:40:03,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:40:03,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:03,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T20:40:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T20:40:03,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T20:40:03,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0670 sec 2024-11-11T20:40:03,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.0750 sec 2024-11-11T20:40:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T20:40:03,182 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:03,189 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-11T20:40:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T20:40:03,192 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:03,195 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:03,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:03,299 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T20:40:03,299 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:03,300 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T20:40:03,300 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:03,300 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T20:40:03,300 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T20:40:03,303 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T20:40:03,303 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:03,303 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,303 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,303 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,303 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,304 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,304 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,304 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:03,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:03,304 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:03,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:03,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T20:40:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:03,349 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-11T20:40:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fedb58c88f25aed82b02e36155b97788 2024-11-11T20:40:03,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. as already flushing 2024-11-11T20:40:03,354 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T20:40:03,354 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:03,354 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,355 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:03,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:03,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/86e3e85203a84a588d62935a362c079b is 50, key is test_row_0/A:col10/1731357602726/Put/seqid=0 2024-11-11T20:40:03,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741942_1118 (size=12301) 2024-11-11T20:40:03,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357663370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:03,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357663381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T20:40:03,764 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/86e3e85203a84a588d62935a362c079b 2024-11-11T20:40:03,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/195134f8aeb5403c88ca0d8e3ed7e9f9 is 50, key is test_row_0/B:col10/1731357602726/Put/seqid=0 2024-11-11T20:40:03,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741943_1119 (size=12301) 2024-11-11T20:40:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T20:40:03,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1731357663881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:03,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48582 deadline: 1731357663891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:04,186 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/195134f8aeb5403c88ca0d8e3ed7e9f9 2024-11-11T20:40:04,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/2eabd39d85a54326adda4ad2b18dcedd is 50, key is test_row_0/C:col10/1731357602726/Put/seqid=0 2024-11-11T20:40:04,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741944_1120 (size=12301) 2024-11-11T20:40:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T20:40:04,606 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/2eabd39d85a54326adda4ad2b18dcedd 2024-11-11T20:40:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/86e3e85203a84a588d62935a362c079b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/86e3e85203a84a588d62935a362c079b 2024-11-11T20:40:04,619 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/86e3e85203a84a588d62935a362c079b, entries=150, sequenceid=530, filesize=12.0 K 2024-11-11T20:40:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/195134f8aeb5403c88ca0d8e3ed7e9f9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/195134f8aeb5403c88ca0d8e3ed7e9f9 2024-11-11T20:40:04,625 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/195134f8aeb5403c88ca0d8e3ed7e9f9, entries=150, sequenceid=530, filesize=12.0 K 2024-11-11T20:40:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/2eabd39d85a54326adda4ad2b18dcedd as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2eabd39d85a54326adda4ad2b18dcedd 2024-11-11T20:40:04,631 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2eabd39d85a54326adda4ad2b18dcedd, entries=150, sequenceid=530, filesize=12.0 K 2024-11-11T20:40:04,632 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for fedb58c88f25aed82b02e36155b97788 in 1284ms, sequenceid=530, compaction requested=true 2024-11-11T20:40:04,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for fedb58c88f25aed82b02e36155b97788: 2024-11-11T20:40:04,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:04,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-11T20:40:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-11T20:40:04,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T20:40:04,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4380 sec 2024-11-11T20:40:04,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4460 sec 2024-11-11T20:40:04,894 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T20:40:04,894 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:04,894 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:04,894 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:04,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:04,903 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T20:40:04,904 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:04,904 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:04,904 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:04,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:05,051 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:40:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T20:40:05,341 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:08,045 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T20:40:08,045 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:08,045 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:08,045 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:08,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:08,073 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T20:40:08,073 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:08,073 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:08,073 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:08,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3814 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4006 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1781 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5336 rows 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1784 2024-11-11T20:40:08,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5348 rows 2024-11-11T20:40:08,075 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:40:08,075 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:40:08,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:08,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:08,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:08,081 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:40:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:08,088 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357608087"}]},"ts":"1731357608087"} 2024-11-11T20:40:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T20:40:08,090 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:40:08,090 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:40:08,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:40:08,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fedb58c88f25aed82b02e36155b97788, UNASSIGN}] 2024-11-11T20:40:08,097 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fedb58c88f25aed82b02e36155b97788, UNASSIGN 2024-11-11T20:40:08,098 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fedb58c88f25aed82b02e36155b97788, regionState=CLOSING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:08,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fedb58c88f25aed82b02e36155b97788, UNASSIGN because future has completed 2024-11-11T20:40:08,101 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:40:08,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:40:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T20:40:08,260 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close fedb58c88f25aed82b02e36155b97788 2024-11-11T20:40:08,260 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:40:08,261 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing fedb58c88f25aed82b02e36155b97788, disabling compactions & flushes 2024-11-11T20:40:08,261 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:08,261 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:08,261 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. after waiting 0 ms 2024-11-11T20:40:08,261 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:08,262 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing fedb58c88f25aed82b02e36155b97788 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T20:40:08,269 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/3d11dc1afe14487d8081ef2d7feb9162 is 50, key is test_row_0/A:col10/1731357604892/Put/seqid=0 2024-11-11T20:40:08,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741945_1121 (size=9857) 2024-11-11T20:40:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T20:40:08,675 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/3d11dc1afe14487d8081ef2d7feb9162 2024-11-11T20:40:08,688 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/68cb6fc7fc9742859e965d292a5ea67a is 50, key is test_row_0/B:col10/1731357604892/Put/seqid=0 2024-11-11T20:40:08,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741946_1122 (size=9857) 2024-11-11T20:40:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T20:40:09,094 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/68cb6fc7fc9742859e965d292a5ea67a 2024-11-11T20:40:09,107 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/e79c3293bb454ab9b1bd3b525ef90e98 is 50, key is test_row_0/C:col10/1731357604892/Put/seqid=0 2024-11-11T20:40:09,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741947_1123 (size=9857) 2024-11-11T20:40:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T20:40:09,512 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/e79c3293bb454ab9b1bd3b525ef90e98 2024-11-11T20:40:09,525 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/A/3d11dc1afe14487d8081ef2d7feb9162 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/3d11dc1afe14487d8081ef2d7feb9162 2024-11-11T20:40:09,529 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/3d11dc1afe14487d8081ef2d7feb9162, entries=100, sequenceid=538, filesize=9.6 K 2024-11-11T20:40:09,530 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/B/68cb6fc7fc9742859e965d292a5ea67a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/68cb6fc7fc9742859e965d292a5ea67a 2024-11-11T20:40:09,535 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/68cb6fc7fc9742859e965d292a5ea67a, entries=100, sequenceid=538, filesize=9.6 K 2024-11-11T20:40:09,536 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/.tmp/C/e79c3293bb454ab9b1bd3b525ef90e98 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/e79c3293bb454ab9b1bd3b525ef90e98 2024-11-11T20:40:09,540 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/e79c3293bb454ab9b1bd3b525ef90e98, entries=100, sequenceid=538, filesize=9.6 K 2024-11-11T20:40:09,541 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fedb58c88f25aed82b02e36155b97788 in 1279ms, sequenceid=538, compaction requested=true 2024-11-11T20:40:09,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ab41baa101484bb4b502095ed23a62ac, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d906ad94c4714b4ea866c2a33456ceff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b29ac9133b9740d0993018fc9400919c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e09f90fb0aac402d99dd8d8d5ad1f6ae, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dd9cc8030f574e59a078c5c5d3ad2930, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/5df1d43025b7456a8c5361a743a8ad22, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/721629911c654800a52c5b9a0d3d562a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/6d93cc8439da45f997b01a2a6c2064c0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/afd6353f69f5419d84f37ba0033cf6c3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/72043ec4ec7f4867bc6fe97baa48d709, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/04405569c3354719a5525ca2ca4a4987, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/def87914f09e4b99b7dd618cc562ee2f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e9f069db477e435f8dbddbbc20fc6499, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/694d07f8ca8640928355996999d7e4ff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/434b22ac58504db7a3a2e19f0074547f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/99228532e68442f782fc1c080727789c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e924d318fcfe4f93a06f53990540f1d5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/bec49577a7ff450586822797869e8eb2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e820b5da29b64868ab6b9d0fe5749f06, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d9f4c2a9577a4836a71b767659d05426, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b4c43d2480704365b388eb39a0d4530e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/fe16f212762541a287bd1b36679d85f0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/913a577098014de0aac00d341a8bc0bc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/abc4067db9ef4f32a2c0bfbbdab0ab57, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dfef0f15985d460ea526526efcd745cc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ce4828381e504504b94b262df668f53c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/7f87deabc38c4ac7bfc178049b79d3cb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/35e7953056654d928599e225ddbbbc2d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/f0138c9d40eb4b33bb75eea97a19fc54, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/8dc89b739ed845289f674ab463e69f23, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4ef13e64e6624ec7a427984d6dea1626, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e2f39e77cfb4414c995d343e1318d37b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/eeb023bc6e084b9cba6dac30e61fe23f] to archive 2024-11-11T20:40:09,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:40:09,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ab41baa101484bb4b502095ed23a62ac to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ab41baa101484bb4b502095ed23a62ac 2024-11-11T20:40:09,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d906ad94c4714b4ea866c2a33456ceff to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d906ad94c4714b4ea866c2a33456ceff 2024-11-11T20:40:09,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b29ac9133b9740d0993018fc9400919c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b29ac9133b9740d0993018fc9400919c 2024-11-11T20:40:09,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e09f90fb0aac402d99dd8d8d5ad1f6ae to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e09f90fb0aac402d99dd8d8d5ad1f6ae 2024-11-11T20:40:09,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dd9cc8030f574e59a078c5c5d3ad2930 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dd9cc8030f574e59a078c5c5d3ad2930 2024-11-11T20:40:09,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/5df1d43025b7456a8c5361a743a8ad22 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/5df1d43025b7456a8c5361a743a8ad22 2024-11-11T20:40:09,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/721629911c654800a52c5b9a0d3d562a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/721629911c654800a52c5b9a0d3d562a 2024-11-11T20:40:09,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/6d93cc8439da45f997b01a2a6c2064c0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/6d93cc8439da45f997b01a2a6c2064c0 2024-11-11T20:40:09,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/afd6353f69f5419d84f37ba0033cf6c3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/afd6353f69f5419d84f37ba0033cf6c3 2024-11-11T20:40:09,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/72043ec4ec7f4867bc6fe97baa48d709 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/72043ec4ec7f4867bc6fe97baa48d709 2024-11-11T20:40:09,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/04405569c3354719a5525ca2ca4a4987 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/04405569c3354719a5525ca2ca4a4987 2024-11-11T20:40:09,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/def87914f09e4b99b7dd618cc562ee2f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/def87914f09e4b99b7dd618cc562ee2f 2024-11-11T20:40:09,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e9f069db477e435f8dbddbbc20fc6499 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e9f069db477e435f8dbddbbc20fc6499 2024-11-11T20:40:09,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/694d07f8ca8640928355996999d7e4ff to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/694d07f8ca8640928355996999d7e4ff 2024-11-11T20:40:09,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/434b22ac58504db7a3a2e19f0074547f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/434b22ac58504db7a3a2e19f0074547f 2024-11-11T20:40:09,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/99228532e68442f782fc1c080727789c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/99228532e68442f782fc1c080727789c 2024-11-11T20:40:09,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e924d318fcfe4f93a06f53990540f1d5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e924d318fcfe4f93a06f53990540f1d5 2024-11-11T20:40:09,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/bec49577a7ff450586822797869e8eb2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/bec49577a7ff450586822797869e8eb2 2024-11-11T20:40:09,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e820b5da29b64868ab6b9d0fe5749f06 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e820b5da29b64868ab6b9d0fe5749f06 2024-11-11T20:40:09,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d9f4c2a9577a4836a71b767659d05426 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/d9f4c2a9577a4836a71b767659d05426 2024-11-11T20:40:09,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b4c43d2480704365b388eb39a0d4530e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/b4c43d2480704365b388eb39a0d4530e 2024-11-11T20:40:09,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/fe16f212762541a287bd1b36679d85f0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/fe16f212762541a287bd1b36679d85f0 2024-11-11T20:40:09,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/913a577098014de0aac00d341a8bc0bc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/913a577098014de0aac00d341a8bc0bc 2024-11-11T20:40:09,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/abc4067db9ef4f32a2c0bfbbdab0ab57 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/abc4067db9ef4f32a2c0bfbbdab0ab57 2024-11-11T20:40:09,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dfef0f15985d460ea526526efcd745cc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/dfef0f15985d460ea526526efcd745cc 2024-11-11T20:40:09,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ce4828381e504504b94b262df668f53c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ce4828381e504504b94b262df668f53c 2024-11-11T20:40:09,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/7f87deabc38c4ac7bfc178049b79d3cb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/7f87deabc38c4ac7bfc178049b79d3cb 2024-11-11T20:40:09,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/35e7953056654d928599e225ddbbbc2d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/35e7953056654d928599e225ddbbbc2d 2024-11-11T20:40:09,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/f0138c9d40eb4b33bb75eea97a19fc54 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/f0138c9d40eb4b33bb75eea97a19fc54 2024-11-11T20:40:09,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/8dc89b739ed845289f674ab463e69f23 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/8dc89b739ed845289f674ab463e69f23 2024-11-11T20:40:09,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4ef13e64e6624ec7a427984d6dea1626 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4ef13e64e6624ec7a427984d6dea1626 2024-11-11T20:40:09,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e2f39e77cfb4414c995d343e1318d37b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/e2f39e77cfb4414c995d343e1318d37b 2024-11-11T20:40:09,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/eeb023bc6e084b9cba6dac30e61fe23f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/eeb023bc6e084b9cba6dac30e61fe23f 2024-11-11T20:40:09,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d378195f06d54de9b2b5cf9cf73eeff9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/dad6c08c868d4d2986c5c2a39e4d3641, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5692ef02aa26482592f88368f865e48a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d4ec22421ebb4f759a1f816c9ca9b502, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ee11eae90cce4df992bd3c5c9f21967d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/e54b40f486214c0a8fee194b9ef9ec3b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/fb248d619edc4d6087b60e951e3a6a9a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/02ab58dae98e4c5f8b2607425bb5297e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/836966131bf34b9fbc319e2506100bfe, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6ae0b4f21dad48a6a10e00894aa93b1e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/1cd729b47619419689899242b21aa003, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/095caa38b12443ee8ac05b0392543c12, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5b127794e9084765af20f94619937661, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c198312083f743859b3bef66b1be403c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/60c5590b8d7d43a283f0b30968005c0b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5deddefebbf84c999de98589968ca469, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/7a28b6e8039248418be7e4b39e0e3e2a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/30bdd078074d482f86d03413c16200fe, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8923affb5aaa4cbb87d6fa2a7163ea3e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/65db89395e9b429cafb90a28ad77d34f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/37b28015a36e4395a49cd231fde3042b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ceb91d19f5b642058e42cf15134c2d7b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5f24bbb3a8b74efcb25edb2792c06d15, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/b4f36ec5a72f4a53b6933423b8e717a4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/57e593bf0c394f149750f7fdc0b97147, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c329de2d8ddf4fab91df6a524ca3dabe, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/f74b0d2b54104ce98ec5e30b3a10c2a4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/a8a6ca01e39c48dc98b581ba23d1231f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e03aba358914d278955875c1abb2165, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6c50e424368c48f5bdac5f8ebd396008, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/2df44e796f7a467bab823fbb96f8240c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c33b9dcc790a4f4e9de268256c8d953b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e837bf1cfc74d51a03a35eb60ef6977] to archive 2024-11-11T20:40:09,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:40:09,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d378195f06d54de9b2b5cf9cf73eeff9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d378195f06d54de9b2b5cf9cf73eeff9 2024-11-11T20:40:09,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/dad6c08c868d4d2986c5c2a39e4d3641 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/dad6c08c868d4d2986c5c2a39e4d3641 2024-11-11T20:40:09,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5692ef02aa26482592f88368f865e48a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5692ef02aa26482592f88368f865e48a 2024-11-11T20:40:09,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d4ec22421ebb4f759a1f816c9ca9b502 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d4ec22421ebb4f759a1f816c9ca9b502 2024-11-11T20:40:09,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ee11eae90cce4df992bd3c5c9f21967d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ee11eae90cce4df992bd3c5c9f21967d 2024-11-11T20:40:09,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/e54b40f486214c0a8fee194b9ef9ec3b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/e54b40f486214c0a8fee194b9ef9ec3b 2024-11-11T20:40:09,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/fb248d619edc4d6087b60e951e3a6a9a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/fb248d619edc4d6087b60e951e3a6a9a 2024-11-11T20:40:09,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/02ab58dae98e4c5f8b2607425bb5297e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/02ab58dae98e4c5f8b2607425bb5297e 2024-11-11T20:40:09,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/836966131bf34b9fbc319e2506100bfe to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/836966131bf34b9fbc319e2506100bfe 2024-11-11T20:40:09,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6ae0b4f21dad48a6a10e00894aa93b1e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6ae0b4f21dad48a6a10e00894aa93b1e 2024-11-11T20:40:09,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/1cd729b47619419689899242b21aa003 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/1cd729b47619419689899242b21aa003 2024-11-11T20:40:09,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/095caa38b12443ee8ac05b0392543c12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/095caa38b12443ee8ac05b0392543c12 2024-11-11T20:40:09,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5b127794e9084765af20f94619937661 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5b127794e9084765af20f94619937661 2024-11-11T20:40:09,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c198312083f743859b3bef66b1be403c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c198312083f743859b3bef66b1be403c 2024-11-11T20:40:09,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/60c5590b8d7d43a283f0b30968005c0b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/60c5590b8d7d43a283f0b30968005c0b 2024-11-11T20:40:09,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5deddefebbf84c999de98589968ca469 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5deddefebbf84c999de98589968ca469 2024-11-11T20:40:09,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/7a28b6e8039248418be7e4b39e0e3e2a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/7a28b6e8039248418be7e4b39e0e3e2a 2024-11-11T20:40:09,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/30bdd078074d482f86d03413c16200fe to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/30bdd078074d482f86d03413c16200fe 2024-11-11T20:40:09,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8923affb5aaa4cbb87d6fa2a7163ea3e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8923affb5aaa4cbb87d6fa2a7163ea3e 2024-11-11T20:40:09,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/65db89395e9b429cafb90a28ad77d34f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/65db89395e9b429cafb90a28ad77d34f 2024-11-11T20:40:09,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/37b28015a36e4395a49cd231fde3042b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/37b28015a36e4395a49cd231fde3042b 2024-11-11T20:40:09,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ceb91d19f5b642058e42cf15134c2d7b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/ceb91d19f5b642058e42cf15134c2d7b 2024-11-11T20:40:09,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5f24bbb3a8b74efcb25edb2792c06d15 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/5f24bbb3a8b74efcb25edb2792c06d15 2024-11-11T20:40:09,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/b4f36ec5a72f4a53b6933423b8e717a4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/b4f36ec5a72f4a53b6933423b8e717a4 2024-11-11T20:40:09,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/57e593bf0c394f149750f7fdc0b97147 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/57e593bf0c394f149750f7fdc0b97147 2024-11-11T20:40:09,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c329de2d8ddf4fab91df6a524ca3dabe to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c329de2d8ddf4fab91df6a524ca3dabe 2024-11-11T20:40:09,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/f74b0d2b54104ce98ec5e30b3a10c2a4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/f74b0d2b54104ce98ec5e30b3a10c2a4 2024-11-11T20:40:09,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/a8a6ca01e39c48dc98b581ba23d1231f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/a8a6ca01e39c48dc98b581ba23d1231f 2024-11-11T20:40:09,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e03aba358914d278955875c1abb2165 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e03aba358914d278955875c1abb2165 2024-11-11T20:40:09,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6c50e424368c48f5bdac5f8ebd396008 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/6c50e424368c48f5bdac5f8ebd396008 2024-11-11T20:40:09,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/2df44e796f7a467bab823fbb96f8240c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/2df44e796f7a467bab823fbb96f8240c 2024-11-11T20:40:09,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c33b9dcc790a4f4e9de268256c8d953b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/c33b9dcc790a4f4e9de268256c8d953b 2024-11-11T20:40:09,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e837bf1cfc74d51a03a35eb60ef6977 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/8e837bf1cfc74d51a03a35eb60ef6977 2024-11-11T20:40:09,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/0ad0518b696d420ead376e1d9576dfd3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/740527e569b242a6a329f62e91e51e83, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/77e6160d4db24c9c85bb0eae8775997e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/ba0b7e5e03074f05a83aeaa15aa3c63e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4173c0af9af84d928b9786df9c65dace, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/c142e1f468cd446791e655bd238daefa, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/abb5853dab434741a6411ca8aab091fd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/69ed921a2fc74658b1afed47c3f6d605, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7b17715326254ead93cb1ad1bcb07f39, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/df7dbc2d20964be4873581adc7409e69, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/feb037f937034510bc0cc78c599443a6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7e56bef185364fd5b0aca9dc207037e0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fb5ce47e0d334e17adbc9ad0ab21d6a0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/faa70175d3f94960834fdfe46ed786df, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/72d0397e923548a397e0e82d767d8bd7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/198ff39dc9654fb98ece87bf02de42c0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4f73d5b3fa184e16a667a82137c75205, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/8dc92b8f924c466ea84c8ad4b2d4c147, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/e43c08fed4f74d85ad3783959189a25b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/953a2aac61b54609b690680ee6ae7032, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/9c6b82c1bd9544daabb3773290fce97c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/10d6fa6eb6784c48a2433dd6fbc05d61, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/96b7f769f04e4d78943e235eba32edbb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2399bed7bc10424f8e8aba8e841451ca, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f3eb86ee331248f49875e7b442897778, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/92cfccd59c494de4b14b2c786b2dc6a7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fba1518798ef4a578341923e6a7dd371, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5e2dc77eea8c49d69f26b38aecaf4a99, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/b03a6815dac64d2c82cc198cee92d994, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/152c0b0464334851a4d6937044a98b68, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/bd4c6b1a228e486dac496cb0fc7f3336, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5068e0d63e5a404a97d77353f66211d8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/648d5601dfea4c9abb62af9b8db68634] to archive 2024-11-11T20:40:09,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:40:09,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/0ad0518b696d420ead376e1d9576dfd3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/0ad0518b696d420ead376e1d9576dfd3 2024-11-11T20:40:09,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/740527e569b242a6a329f62e91e51e83 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/740527e569b242a6a329f62e91e51e83 2024-11-11T20:40:09,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/77e6160d4db24c9c85bb0eae8775997e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/77e6160d4db24c9c85bb0eae8775997e 2024-11-11T20:40:09,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/ba0b7e5e03074f05a83aeaa15aa3c63e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/ba0b7e5e03074f05a83aeaa15aa3c63e 2024-11-11T20:40:09,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4173c0af9af84d928b9786df9c65dace to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4173c0af9af84d928b9786df9c65dace 2024-11-11T20:40:09,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/c142e1f468cd446791e655bd238daefa to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/c142e1f468cd446791e655bd238daefa 2024-11-11T20:40:09,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/abb5853dab434741a6411ca8aab091fd to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/abb5853dab434741a6411ca8aab091fd 2024-11-11T20:40:09,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/69ed921a2fc74658b1afed47c3f6d605 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/69ed921a2fc74658b1afed47c3f6d605 2024-11-11T20:40:09,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7b17715326254ead93cb1ad1bcb07f39 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7b17715326254ead93cb1ad1bcb07f39 2024-11-11T20:40:09,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/df7dbc2d20964be4873581adc7409e69 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/df7dbc2d20964be4873581adc7409e69 2024-11-11T20:40:09,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/feb037f937034510bc0cc78c599443a6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/feb037f937034510bc0cc78c599443a6 2024-11-11T20:40:09,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7e56bef185364fd5b0aca9dc207037e0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/7e56bef185364fd5b0aca9dc207037e0 2024-11-11T20:40:09,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fb5ce47e0d334e17adbc9ad0ab21d6a0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fb5ce47e0d334e17adbc9ad0ab21d6a0 2024-11-11T20:40:09,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/faa70175d3f94960834fdfe46ed786df to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/faa70175d3f94960834fdfe46ed786df 2024-11-11T20:40:09,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/72d0397e923548a397e0e82d767d8bd7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/72d0397e923548a397e0e82d767d8bd7 2024-11-11T20:40:09,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/198ff39dc9654fb98ece87bf02de42c0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/198ff39dc9654fb98ece87bf02de42c0 2024-11-11T20:40:09,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4f73d5b3fa184e16a667a82137c75205 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/4f73d5b3fa184e16a667a82137c75205 2024-11-11T20:40:09,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/8dc92b8f924c466ea84c8ad4b2d4c147 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/8dc92b8f924c466ea84c8ad4b2d4c147 2024-11-11T20:40:09,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/e43c08fed4f74d85ad3783959189a25b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/e43c08fed4f74d85ad3783959189a25b 2024-11-11T20:40:09,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/953a2aac61b54609b690680ee6ae7032 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/953a2aac61b54609b690680ee6ae7032 2024-11-11T20:40:09,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/9c6b82c1bd9544daabb3773290fce97c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/9c6b82c1bd9544daabb3773290fce97c 2024-11-11T20:40:09,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/10d6fa6eb6784c48a2433dd6fbc05d61 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/10d6fa6eb6784c48a2433dd6fbc05d61 2024-11-11T20:40:09,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/96b7f769f04e4d78943e235eba32edbb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/96b7f769f04e4d78943e235eba32edbb 2024-11-11T20:40:09,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2399bed7bc10424f8e8aba8e841451ca to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2399bed7bc10424f8e8aba8e841451ca 2024-11-11T20:40:09,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f3eb86ee331248f49875e7b442897778 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f3eb86ee331248f49875e7b442897778 2024-11-11T20:40:09,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/92cfccd59c494de4b14b2c786b2dc6a7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/92cfccd59c494de4b14b2c786b2dc6a7 2024-11-11T20:40:09,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fba1518798ef4a578341923e6a7dd371 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/fba1518798ef4a578341923e6a7dd371 2024-11-11T20:40:09,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5e2dc77eea8c49d69f26b38aecaf4a99 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5e2dc77eea8c49d69f26b38aecaf4a99 2024-11-11T20:40:09,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/b03a6815dac64d2c82cc198cee92d994 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/b03a6815dac64d2c82cc198cee92d994 2024-11-11T20:40:09,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/152c0b0464334851a4d6937044a98b68 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/152c0b0464334851a4d6937044a98b68 2024-11-11T20:40:09,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/bd4c6b1a228e486dac496cb0fc7f3336 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/bd4c6b1a228e486dac496cb0fc7f3336 2024-11-11T20:40:09,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5068e0d63e5a404a97d77353f66211d8 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/5068e0d63e5a404a97d77353f66211d8 2024-11-11T20:40:09,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/648d5601dfea4c9abb62af9b8db68634 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/648d5601dfea4c9abb62af9b8db68634 2024-11-11T20:40:09,696 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/recovered.edits/541.seqid, newMaxSeqId=541, maxSeqId=1 2024-11-11T20:40:09,697 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. 2024-11-11T20:40:09,697 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for fedb58c88f25aed82b02e36155b97788: Waiting for close lock at 1731357608261Running coprocessor pre-close hooks at 1731357608261Disabling compacts and flushes for region at 1731357608261Disabling writes for close at 1731357608261Obtaining lock to block concurrent updates at 1731357608262 (+1 ms)Preparing flush snapshotting stores in fedb58c88f25aed82b02e36155b97788 at 1731357608262Finished memstore snapshotting TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731357608262Flushing stores of TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788. at 1731357608263 (+1 ms)Flushing fedb58c88f25aed82b02e36155b97788/A: creating writer at 1731357608263Flushing fedb58c88f25aed82b02e36155b97788/A: appending metadata at 1731357608268 (+5 ms)Flushing fedb58c88f25aed82b02e36155b97788/A: closing flushed file at 1731357608269 (+1 ms)Flushing fedb58c88f25aed82b02e36155b97788/B: creating writer at 1731357608683 (+414 ms)Flushing fedb58c88f25aed82b02e36155b97788/B: appending metadata at 1731357608687 (+4 ms)Flushing fedb58c88f25aed82b02e36155b97788/B: closing flushed file at 1731357608687Flushing fedb58c88f25aed82b02e36155b97788/C: creating writer at 1731357609104 (+417 ms)Flushing fedb58c88f25aed82b02e36155b97788/C: appending metadata at 1731357609106 (+2 ms)Flushing fedb58c88f25aed82b02e36155b97788/C: closing flushed file at 1731357609107 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1079f711: reopening flushed file at 1731357609523 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54a078e9: reopening flushed file at 1731357609529 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a4eb714: reopening flushed file at 1731357609535 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fedb58c88f25aed82b02e36155b97788 in 1279ms, sequenceid=538, compaction requested=true at 1731357609541 (+6 ms)Writing region close event to WAL at 1731357609693 (+152 ms)Running coprocessor post-close hooks at 1731357609697 (+4 ms)Closed at 1731357609697 2024-11-11T20:40:09,699 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed fedb58c88f25aed82b02e36155b97788 2024-11-11T20:40:09,700 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fedb58c88f25aed82b02e36155b97788, regionState=CLOSED 2024-11-11T20:40:09,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:40:09,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T20:40:09,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure fedb58c88f25aed82b02e36155b97788, server=51ca66f7ee3c,44271,1731357576703 in 1.6010 sec 2024-11-11T20:40:09,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-11T20:40:09,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fedb58c88f25aed82b02e36155b97788, UNASSIGN in 1.6080 sec 2024-11-11T20:40:09,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T20:40:09,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6160 sec 2024-11-11T20:40:09,710 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357609710"}]},"ts":"1731357609710"} 2024-11-11T20:40:09,712 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:40:09,712 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:40:09,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6300 sec 2024-11-11T20:40:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T20:40:10,241 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:10,244 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:40:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:10,249 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T20:40:10,252 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:10,256 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788 2024-11-11T20:40:10,260 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/recovered.edits] 2024-11-11T20:40:10,264 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/3d11dc1afe14487d8081ef2d7feb9162 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/3d11dc1afe14487d8081ef2d7feb9162 2024-11-11T20:40:10,265 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4dc26f74b36342a99582387b0ab88057 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/4dc26f74b36342a99582387b0ab88057 2024-11-11T20:40:10,267 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/86e3e85203a84a588d62935a362c079b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/86e3e85203a84a588d62935a362c079b 2024-11-11T20:40:10,268 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ce1a10ea4c4b4e44b089c17631a7d6af to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/A/ce1a10ea4c4b4e44b089c17631a7d6af 2024-11-11T20:40:10,270 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/195134f8aeb5403c88ca0d8e3ed7e9f9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/195134f8aeb5403c88ca0d8e3ed7e9f9 2024-11-11T20:40:10,272 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/19c5d0ea7da9416ea2d560bddcf982a8 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/19c5d0ea7da9416ea2d560bddcf982a8 2024-11-11T20:40:10,273 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/68cb6fc7fc9742859e965d292a5ea67a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/68cb6fc7fc9742859e965d292a5ea67a 2024-11-11T20:40:10,274 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d8eb541d74c3472d9fbf74dfda1e49f8 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/B/d8eb541d74c3472d9fbf74dfda1e49f8 2024-11-11T20:40:10,277 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2eabd39d85a54326adda4ad2b18dcedd to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/2eabd39d85a54326adda4ad2b18dcedd 2024-11-11T20:40:10,278 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/b05cfd5cb34a4ff2b6ca6010589787ea to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/b05cfd5cb34a4ff2b6ca6010589787ea 2024-11-11T20:40:10,279 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/e79c3293bb454ab9b1bd3b525ef90e98 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/e79c3293bb454ab9b1bd3b525ef90e98 2024-11-11T20:40:10,280 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f19e5c20a22642df8bfb95d5b81b9999 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/C/f19e5c20a22642df8bfb95d5b81b9999 2024-11-11T20:40:10,283 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/recovered.edits/541.seqid to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788/recovered.edits/541.seqid 2024-11-11T20:40:10,283 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fedb58c88f25aed82b02e36155b97788 2024-11-11T20:40:10,283 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:40:10,289 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:10,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T20:40:10,298 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:40:10,327 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:40:10,328 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:10,328 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:40:10,328 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731357610328"}]},"ts":"9223372036854775807"} 2024-11-11T20:40:10,334 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:40:10,334 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fedb58c88f25aed82b02e36155b97788, NAME => 'TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:40:10,334 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:40:10,335 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731357610334"}]},"ts":"9223372036854775807"} 2024-11-11T20:40:10,340 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:40:10,342 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:10,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 97 msec 2024-11-11T20:40:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T20:40:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:40:10,362 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:10,376 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 212) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;51ca66f7ee3c:44271-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2095157064_22 at /127.0.0.1:43646 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 259) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4221 (was 5162) 2024-11-11T20:40:10,385 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=4220 2024-11-11T20:40:10,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:40:10,387 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:40:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:10,390 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:40:10,390 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:10,390 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-11T20:40:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:40:10,391 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:40:10,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741948_1124 (size=959) 2024-11-11T20:40:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:40:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:40:10,800 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30 2024-11-11T20:40:10,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741949_1125 (size=53) 2024-11-11T20:40:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:40:11,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:40:11,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3c30a36b246285db6085785ae9fb5d59, disabling compactions & flushes 2024-11-11T20:40:11,208 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:11,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:11,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. after waiting 0 ms 2024-11-11T20:40:11,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:11,208 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:11,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3c30a36b246285db6085785ae9fb5d59: Waiting for close lock at 1731357611207Disabling compacts and flushes for region at 1731357611207Disabling writes for close at 1731357611208 (+1 ms)Writing region close event to WAL at 1731357611208Closed at 1731357611208 2024-11-11T20:40:11,210 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:40:11,211 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731357611210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731357611210"}]},"ts":"1731357611210"} 2024-11-11T20:40:11,214 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:40:11,216 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:40:11,216 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357611216"}]},"ts":"1731357611216"} 2024-11-11T20:40:11,219 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:40:11,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, ASSIGN}] 2024-11-11T20:40:11,221 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, ASSIGN 2024-11-11T20:40:11,222 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44271,1731357576703; forceNewPlan=false, retain=false 2024-11-11T20:40:11,373 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=3c30a36b246285db6085785ae9fb5d59, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:11,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, ASSIGN because future has completed 2024-11-11T20:40:11,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:40:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:40:11,537 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:11,538 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:40:11,538 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,538 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:40:11,538 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,538 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,539 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,541 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c30a36b246285db6085785ae9fb5d59 columnFamilyName A 2024-11-11T20:40:11,542 DEBUG [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:11,542 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(327): Store=3c30a36b246285db6085785ae9fb5d59/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:40:11,542 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,543 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c30a36b246285db6085785ae9fb5d59 columnFamilyName B 2024-11-11T20:40:11,543 DEBUG [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:11,544 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(327): Store=3c30a36b246285db6085785ae9fb5d59/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:40:11,544 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,545 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c30a36b246285db6085785ae9fb5d59 columnFamilyName C 2024-11-11T20:40:11,545 DEBUG [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:11,545 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(327): Store=3c30a36b246285db6085785ae9fb5d59/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:40:11,545 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,546 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,546 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,547 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,547 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,548 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:40:11,549 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,551 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:40:11,552 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 3c30a36b246285db6085785ae9fb5d59; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68441341, jitterRate=0.019855454564094543}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:40:11,552 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:11,553 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 3c30a36b246285db6085785ae9fb5d59: Running coprocessor pre-open hook at 1731357611538Writing region info on filesystem at 1731357611538Initializing all the Stores at 1731357611539 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357611539Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357611539Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357611539Cleaning up temporary data from old regions at 1731357611547 (+8 ms)Running coprocessor post-open hooks at 1731357611552 (+5 ms)Region opened successfully at 1731357611553 (+1 ms) 2024-11-11T20:40:11,554 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., pid=34, masterSystemTime=1731357611533 2024-11-11T20:40:11,556 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:11,556 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:11,556 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=3c30a36b246285db6085785ae9fb5d59, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:11,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:40:11,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-11T20:40:11,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 in 179 msec 2024-11-11T20:40:11,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-11T20:40:11,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, ASSIGN in 342 msec 2024-11-11T20:40:11,564 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:40:11,564 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357611564"}]},"ts":"1731357611564"} 2024-11-11T20:40:11,566 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:40:11,567 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:40:11,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-11T20:40:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:40:12,543 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:12,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcb2494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:12,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:12,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:12,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57af870e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:12,552 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b76b47a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:12,554 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:12,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:12,560 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:40:12,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,35885,1731357576039 2024-11-11T20:40:12,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@73b9f97d 2024-11-11T20:40:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:40:12,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:40:12,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:40:12,570 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:40:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:12,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741950_1126 (size=995) 2024-11-11T20:40:12,618 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T20:40:12,618 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T20:40:12,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:40:12,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, REOPEN/MOVE}] 2024-11-11T20:40:12,637 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, REOPEN/MOVE 2024-11-11T20:40:12,639 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=3c30a36b246285db6085785ae9fb5d59, regionState=CLOSING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:12,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, REOPEN/MOVE because future has completed 2024-11-11T20:40:12,650 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35885 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=51ca66f7ee3c,44271,1731357576703, table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T20:40:12,659 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:40:12,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:40:12,814 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:12,814 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:40:12,814 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 3c30a36b246285db6085785ae9fb5d59, disabling compactions & flushes 2024-11-11T20:40:12,815 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:12,815 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:12,815 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. after waiting 0 ms 2024-11-11T20:40:12,815 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:12,820 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:40:12,821 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:12,821 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 3c30a36b246285db6085785ae9fb5d59: Waiting for close lock at 1731357612814Running coprocessor pre-close hooks at 1731357612814Disabling compacts and flushes for region at 1731357612814Disabling writes for close at 1731357612815 (+1 ms)Writing region close event to WAL at 1731357612815Running coprocessor post-close hooks at 1731357612820 (+5 ms)Closed at 1731357612821 (+1 ms) 2024-11-11T20:40:12,821 WARN [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 3c30a36b246285db6085785ae9fb5d59 to self. 2024-11-11T20:40:12,823 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:12,824 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=3c30a36b246285db6085785ae9fb5d59, regionState=CLOSED 2024-11-11T20:40:12,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:40:12,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-11T20:40:12,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 in 168 msec 2024-11-11T20:40:12,829 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, REOPEN/MOVE; state=CLOSED, location=51ca66f7ee3c,44271,1731357576703; forceNewPlan=false, retain=true 2024-11-11T20:40:12,980 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=3c30a36b246285db6085785ae9fb5d59, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:12,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, REOPEN/MOVE because future has completed 2024-11-11T20:40:12,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:40:13,140 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:13,141 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:40:13,141 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,142 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:40:13,142 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,142 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,148 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,155 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c30a36b246285db6085785ae9fb5d59 columnFamilyName A 2024-11-11T20:40:13,157 DEBUG [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:13,157 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(327): Store=3c30a36b246285db6085785ae9fb5d59/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:40:13,158 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,158 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c30a36b246285db6085785ae9fb5d59 columnFamilyName B 2024-11-11T20:40:13,158 DEBUG [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:13,159 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(327): Store=3c30a36b246285db6085785ae9fb5d59/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:40:13,159 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,160 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c30a36b246285db6085785ae9fb5d59 columnFamilyName C 2024-11-11T20:40:13,160 DEBUG [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:13,160 INFO [StoreOpener-3c30a36b246285db6085785ae9fb5d59-1 {}] regionserver.HStore(327): Store=3c30a36b246285db6085785ae9fb5d59/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:40:13,160 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,161 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,162 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,163 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,163 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,163 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:40:13,165 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,166 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 3c30a36b246285db6085785ae9fb5d59; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73793365, jitterRate=0.09960682690143585}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:40:13,166 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,167 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 3c30a36b246285db6085785ae9fb5d59: Running coprocessor pre-open hook at 1731357613142Writing region info on filesystem at 1731357613142Initializing all the Stores at 1731357613144 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357613144Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357613144Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357613145 (+1 ms)Cleaning up temporary data from old regions at 1731357613163 (+18 ms)Running coprocessor post-open hooks at 1731357613166 (+3 ms)Region opened successfully at 1731357613167 (+1 ms) 2024-11-11T20:40:13,168 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., pid=39, masterSystemTime=1731357613134 2024-11-11T20:40:13,170 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:13,170 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:13,171 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=3c30a36b246285db6085785ae9fb5d59, regionState=OPEN, openSeqNum=5, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:40:13,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-11T20:40:13,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 in 191 msec 2024-11-11T20:40:13,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-11T20:40:13,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, REOPEN/MOVE in 541 msec 2024-11-11T20:40:13,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T20:40:13,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 555 msec 2024-11-11T20:40:13,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 610 msec 2024-11-11T20:40:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-11T20:40:13,188 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:13,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:13,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:13,190 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd6a820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:13,193 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799faa56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:13,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:13,195 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3fb0ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:13,196 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754a34a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:13,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:13,198 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3678386c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:13,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:13,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2647a7aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:13,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:13,202 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:13,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:13,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:13,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280b3d93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:13,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:13,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:13,203 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754ca93a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:13,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:13,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33925b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:13,207 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cff4084, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:13,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:13,209 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4888ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:13,210 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69247b06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:13,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:13,212 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3677ca3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:13,213 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3202d9a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:13,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:13,215 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c0cc05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:13,216 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0dd047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,217 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:13,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:13,218 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724f3c5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:13,220 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364eb416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:13,221 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:13,221 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:13,221 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:13,221 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:13,221 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:13,223 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:13,223 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:13,223 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:13,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:13,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:13,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:13,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:13,225 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:13,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:13,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:13,226 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-11T20:40:13,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:13,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:13,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:13,228 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:40:13,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:13,228 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:13,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:13,229 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:13,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:13,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:40:13,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:40:13,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:40:13,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:40:13,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:40:13,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:40:13,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:40:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:40:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:40:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:13,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:40:13,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dfb93484786547cc8d0bbb57c7c35d05_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357613243/Put/seqid=0 2024-11-11T20:40:13,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741951_1127 (size=17034) 2024-11-11T20:40:13,312 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:13,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357673306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357673309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357673309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,320 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dfb93484786547cc8d0bbb57c7c35d05_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dfb93484786547cc8d0bbb57c7c35d05_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,325 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/68ba1e1967124b169338ee8799a835ff, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:40:13,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357673322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357673322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/68ba1e1967124b169338ee8799a835ff is 175, key is test_row_0/A:col10/1731357613243/Put/seqid=0 2024-11-11T20:40:13,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741952_1128 (size=48139) 2024-11-11T20:40:13,345 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/68ba1e1967124b169338ee8799a835ff 2024-11-11T20:40:13,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/67559b672b1b4699af6a519953ee5d63 is 50, key is test_row_0/B:col10/1731357613243/Put/seqid=0 2024-11-11T20:40:13,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T20:40:13,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:13,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:13,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:13,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:13,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:13,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741953_1129 (size=12001) 2024-11-11T20:40:13,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/67559b672b1b4699af6a519953ee5d63 2024-11-11T20:40:13,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357673421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357673422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/260d9934c7084b5898da7233be6f91a4 is 50, key is test_row_0/C:col10/1731357613243/Put/seqid=0 2024-11-11T20:40:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357673423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741954_1130 (size=12001) 2024-11-11T20:40:13,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357673441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/260d9934c7084b5898da7233be6f91a4 2024-11-11T20:40:13,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357673441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/68ba1e1967124b169338ee8799a835ff as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/68ba1e1967124b169338ee8799a835ff 2024-11-11T20:40:13,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/68ba1e1967124b169338ee8799a835ff, entries=250, sequenceid=16, filesize=47.0 K 2024-11-11T20:40:13,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/67559b672b1b4699af6a519953ee5d63 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/67559b672b1b4699af6a519953ee5d63 2024-11-11T20:40:13,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/67559b672b1b4699af6a519953ee5d63, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T20:40:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/260d9934c7084b5898da7233be6f91a4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/260d9934c7084b5898da7233be6f91a4 2024-11-11T20:40:13,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/260d9934c7084b5898da7233be6f91a4, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T20:40:13,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3c30a36b246285db6085785ae9fb5d59 in 253ms, sequenceid=16, compaction requested=false 2024-11-11T20:40:13,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:13,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T20:40:13,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:13,539 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:40:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:40:13,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f16f5e2f8b514cacb69a30b6411d86f9_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357613309/Put/seqid=0 2024-11-11T20:40:13,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741955_1131 (size=12154) 2024-11-11T20:40:13,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:13,604 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f16f5e2f8b514cacb69a30b6411d86f9_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f16f5e2f8b514cacb69a30b6411d86f9_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/930fcf4fed594c08a2524ed8ad2fd95a, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:13,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/930fcf4fed594c08a2524ed8ad2fd95a is 175, key is test_row_0/A:col10/1731357613309/Put/seqid=0 2024-11-11T20:40:13,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741956_1132 (size=30955) 2024-11-11T20:40:13,628 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/930fcf4fed594c08a2524ed8ad2fd95a 2024-11-11T20:40:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/ca1064fb3fa54ee18d613d239348cc68 is 50, key is test_row_0/B:col10/1731357613309/Put/seqid=0 2024-11-11T20:40:13,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357673658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357673665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357673666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357673669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357673671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741957_1133 (size=12001) 2024-11-11T20:40:13,683 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/ca1064fb3fa54ee18d613d239348cc68 2024-11-11T20:40:13,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e7c9b293afdf41c59c3a55d6949c30e9 is 50, key is test_row_0/C:col10/1731357613309/Put/seqid=0 2024-11-11T20:40:13,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741958_1134 (size=12001) 2024-11-11T20:40:13,734 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e7c9b293afdf41c59c3a55d6949c30e9 2024-11-11T20:40:13,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/930fcf4fed594c08a2524ed8ad2fd95a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/930fcf4fed594c08a2524ed8ad2fd95a 2024-11-11T20:40:13,751 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/930fcf4fed594c08a2524ed8ad2fd95a, entries=150, sequenceid=40, filesize=30.2 K 2024-11-11T20:40:13,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/ca1064fb3fa54ee18d613d239348cc68 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ca1064fb3fa54ee18d613d239348cc68 2024-11-11T20:40:13,760 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ca1064fb3fa54ee18d613d239348cc68, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T20:40:13,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e7c9b293afdf41c59c3a55d6949c30e9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e7c9b293afdf41c59c3a55d6949c30e9 2024-11-11T20:40:13,772 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e7c9b293afdf41c59c3a55d6949c30e9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T20:40:13,778 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3c30a36b246285db6085785ae9fb5d59 in 240ms, sequenceid=40, compaction requested=false 2024-11-11T20:40:13,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:13,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-11T20:40:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-11T20:40:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,784 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:13,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T20:40:13,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-11T20:40:13,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 554 msec 2024-11-11T20:40:13,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 561 msec 2024-11-11T20:40:13,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118096c0cbc87b4840a3925e3da70e6efd_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357613665/Put/seqid=0 2024-11-11T20:40:13,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741959_1135 (size=14594) 2024-11-11T20:40:13,831 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:13,843 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118096c0cbc87b4840a3925e3da70e6efd_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118096c0cbc87b4840a3925e3da70e6efd_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:13,846 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/987f6e75deb74048b5383af7d967e7af, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:13,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/987f6e75deb74048b5383af7d967e7af is 175, key is test_row_0/A:col10/1731357613665/Put/seqid=0 2024-11-11T20:40:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:40:13,853 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:13,853 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T20:40:13,855 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:40:13,856 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:13,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357673839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357673843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357673844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357673848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357673855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741960_1136 (size=39549) 2024-11-11T20:40:13,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/987f6e75deb74048b5383af7d967e7af 2024-11-11T20:40:13,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/6a9bfdfe450745f098d209dc0f2c446d is 50, key is test_row_0/B:col10/1731357613665/Put/seqid=0 2024-11-11T20:40:13,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741961_1137 (size=12001) 2024-11-11T20:40:13,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/6a9bfdfe450745f098d209dc0f2c446d 2024-11-11T20:40:13,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/f42b36173fb940dd82921c161d5b0f28 is 50, key is test_row_0/C:col10/1731357613665/Put/seqid=0 2024-11-11T20:40:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:40:13,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357673971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357673971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357673971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357673971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357673971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741962_1138 (size=12001) 2024-11-11T20:40:14,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:40:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357674191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357674191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357674192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357674192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357674192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:14,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:14,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/f42b36173fb940dd82921c161d5b0f28 2024-11-11T20:40:14,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/987f6e75deb74048b5383af7d967e7af as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/987f6e75deb74048b5383af7d967e7af 2024-11-11T20:40:14,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/987f6e75deb74048b5383af7d967e7af, entries=200, sequenceid=53, filesize=38.6 K 2024-11-11T20:40:14,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/6a9bfdfe450745f098d209dc0f2c446d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/6a9bfdfe450745f098d209dc0f2c446d 2024-11-11T20:40:14,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/6a9bfdfe450745f098d209dc0f2c446d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-11T20:40:14,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/f42b36173fb940dd82921c161d5b0f28 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/f42b36173fb940dd82921c161d5b0f28 2024-11-11T20:40:14,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:14,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:40:14,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/f42b36173fb940dd82921c161d5b0f28, entries=150, sequenceid=53, filesize=11.7 K 2024-11-11T20:40:14,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3c30a36b246285db6085785ae9fb5d59 in 711ms, sequenceid=53, compaction requested=true 2024-11-11T20:40:14,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:14,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:14,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:14,495 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:14,496 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:14,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:14,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:14,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:14,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:14,497 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:14,497 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/A is initiating minor compaction (all files) 2024-11-11T20:40:14,497 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/A in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,497 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/68ba1e1967124b169338ee8799a835ff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/930fcf4fed594c08a2524ed8ad2fd95a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/987f6e75deb74048b5383af7d967e7af] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=115.9 K 2024-11-11T20:40:14,497 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,497 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/68ba1e1967124b169338ee8799a835ff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/930fcf4fed594c08a2524ed8ad2fd95a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/987f6e75deb74048b5383af7d967e7af] 2024-11-11T20:40:14,498 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ba1e1967124b169338ee8799a835ff, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731357613238 2024-11-11T20:40:14,499 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 930fcf4fed594c08a2524ed8ad2fd95a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731357613283 2024-11-11T20:40:14,499 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:14,499 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/B is initiating minor compaction (all files) 2024-11-11T20:40:14,499 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/B in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,499 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/67559b672b1b4699af6a519953ee5d63, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ca1064fb3fa54ee18d613d239348cc68, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/6a9bfdfe450745f098d209dc0f2c446d] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=35.2 K 2024-11-11T20:40:14,501 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 987f6e75deb74048b5383af7d967e7af, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731357613659 2024-11-11T20:40:14,501 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 67559b672b1b4699af6a519953ee5d63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731357613238 2024-11-11T20:40:14,503 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1064fb3fa54ee18d613d239348cc68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731357613283 2024-11-11T20:40:14,503 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a9bfdfe450745f098d209dc0f2c446d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731357613659 2024-11-11T20:40:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:14,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:14,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T20:40:14,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357674515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357674517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357674517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357674518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357674520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,531 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:14,539 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:14,539 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/b7d7a1d35fc14ca1998ab0e91615b7df is 50, key is test_row_0/B:col10/1731357613665/Put/seqid=0 2024-11-11T20:40:14,548 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111bd5eb3ec328b45e7bd636e3fe9690c98_3c30a36b246285db6085785ae9fb5d59 store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:14,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111312aa1dcfaf46568537c9101b794ef6_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357614502/Put/seqid=0 2024-11-11T20:40:14,557 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111bd5eb3ec328b45e7bd636e3fe9690c98_3c30a36b246285db6085785ae9fb5d59, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:14,558 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bd5eb3ec328b45e7bd636e3fe9690c98_3c30a36b246285db6085785ae9fb5d59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:14,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741963_1139 (size=12104) 2024-11-11T20:40:14,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741964_1140 (size=17034) 2024-11-11T20:40:14,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741965_1141 (size=4469) 2024-11-11T20:40:14,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:14,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:14,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357674631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357674631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357674632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357674632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357674632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357674841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357674841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357674841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357674841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357674845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:14,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:14,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:14,990 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/b7d7a1d35fc14ca1998ab0e91615b7df as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b7d7a1d35fc14ca1998ab0e91615b7df 2024-11-11T20:40:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:40:14,995 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:14,998 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/B of 3c30a36b246285db6085785ae9fb5d59 into b7d7a1d35fc14ca1998ab0e91615b7df(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:14,998 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:14,998 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/B, priority=13, startTime=1731357614496; duration=0sec 2024-11-11T20:40:14,999 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:14,999 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:15,000 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:15,001 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:15,001 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/C is initiating minor compaction (all files) 2024-11-11T20:40:15,002 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/C in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:15,002 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/260d9934c7084b5898da7233be6f91a4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e7c9b293afdf41c59c3a55d6949c30e9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/f42b36173fb940dd82921c161d5b0f28] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=35.2 K 2024-11-11T20:40:15,002 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 260d9934c7084b5898da7233be6f91a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731357613238 2024-11-11T20:40:15,002 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111312aa1dcfaf46568537c9101b794ef6_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111312aa1dcfaf46568537c9101b794ef6_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:15,003 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c9b293afdf41c59c3a55d6949c30e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731357613283 2024-11-11T20:40:15,003 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f42b36173fb940dd82921c161d5b0f28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731357613659 2024-11-11T20:40:15,004 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/6f324b862f4b4db8bd69f3bf0e704c37, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:15,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/6f324b862f4b4db8bd69f3bf0e704c37 is 175, key is test_row_0/A:col10/1731357614502/Put/seqid=0 2024-11-11T20:40:15,008 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#A#compaction#120 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:15,009 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b70cf6f663a94173b55d29be21d50c0c is 175, key is test_row_0/A:col10/1731357613665/Put/seqid=0 2024-11-11T20:40:15,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741966_1142 (size=48139) 2024-11-11T20:40:15,015 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/6f324b862f4b4db8bd69f3bf0e704c37 2024-11-11T20:40:15,018 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:15,018 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/3234a3da5587434b921550651bb0fdcb is 50, key is test_row_0/C:col10/1731357613665/Put/seqid=0 2024-11-11T20:40:15,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741967_1143 (size=31058) 2024-11-11T20:40:15,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741968_1144 (size=12104) 2024-11-11T20:40:15,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/fdb60ea61b7d4883994c3d8d0b47fa09 is 50, key is test_row_0/B:col10/1731357614502/Put/seqid=0 2024-11-11T20:40:15,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741969_1145 (size=12001) 2024-11-11T20:40:15,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/fdb60ea61b7d4883994c3d8d0b47fa09 2024-11-11T20:40:15,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:15,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:15,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/1b0c81f9d4d1426cb664170eff0f9b91 is 50, key is test_row_0/C:col10/1731357614502/Put/seqid=0 2024-11-11T20:40:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:15,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741970_1146 (size=12001) 2024-11-11T20:40:15,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357675151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357675151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357675151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357675151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357675161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:15,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:15,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:15,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:15,260 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:40:15,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:15,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:15,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:15,434 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b70cf6f663a94173b55d29be21d50c0c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b70cf6f663a94173b55d29be21d50c0c 2024-11-11T20:40:15,440 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/3234a3da5587434b921550651bb0fdcb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/3234a3da5587434b921550651bb0fdcb 2024-11-11T20:40:15,445 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/A of 3c30a36b246285db6085785ae9fb5d59 into b70cf6f663a94173b55d29be21d50c0c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:15,445 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:15,445 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/A, priority=13, startTime=1731357614495; duration=0sec 2024-11-11T20:40:15,445 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:15,445 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:15,448 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/C of 3c30a36b246285db6085785ae9fb5d59 into 3234a3da5587434b921550651bb0fdcb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:15,448 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:15,448 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/C, priority=13, startTime=1731357614496; duration=0sec 2024-11-11T20:40:15,448 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:15,448 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:15,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/1b0c81f9d4d1426cb664170eff0f9b91 2024-11-11T20:40:15,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/6f324b862f4b4db8bd69f3bf0e704c37 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/6f324b862f4b4db8bd69f3bf0e704c37 2024-11-11T20:40:15,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/6f324b862f4b4db8bd69f3bf0e704c37, entries=250, sequenceid=78, filesize=47.0 K 2024-11-11T20:40:15,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/fdb60ea61b7d4883994c3d8d0b47fa09 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fdb60ea61b7d4883994c3d8d0b47fa09 2024-11-11T20:40:15,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fdb60ea61b7d4883994c3d8d0b47fa09, entries=150, sequenceid=78, filesize=11.7 K 2024-11-11T20:40:15,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/1b0c81f9d4d1426cb664170eff0f9b91 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1b0c81f9d4d1426cb664170eff0f9b91 2024-11-11T20:40:15,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1b0c81f9d4d1426cb664170eff0f9b91, entries=150, sequenceid=78, filesize=11.7 K 2024-11-11T20:40:15,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3c30a36b246285db6085785ae9fb5d59 in 1054ms, sequenceid=78, compaction requested=false 2024-11-11T20:40:15,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:15,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:40:15,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:15,564 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:40:15,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111eb3ca1836ffe483489399aaa31d5cafc_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357614512/Put/seqid=0 2024-11-11T20:40:15,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741971_1147 (size=12154) 2024-11-11T20:40:15,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:15,667 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111eb3ca1836ffe483489399aaa31d5cafc_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111eb3ca1836ffe483489399aaa31d5cafc_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:15,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:15,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/879f6b28b4054aee88c62163acb2bb5d, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:15,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/879f6b28b4054aee88c62163acb2bb5d is 175, key is test_row_0/A:col10/1731357614512/Put/seqid=0 2024-11-11T20:40:15,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741972_1148 (size=30955) 2024-11-11T20:40:15,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357675710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357675712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357675719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357675733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357675735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357675841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357675841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357675841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357675851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:15,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357675860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:15,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:40:16,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357676060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357676060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357676061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357676072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357676072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,126 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/879f6b28b4054aee88c62163acb2bb5d 2024-11-11T20:40:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/c578dced20324c579042babe0830bf44 is 50, key is test_row_0/B:col10/1731357614512/Put/seqid=0 2024-11-11T20:40:16,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741973_1149 (size=12001) 2024-11-11T20:40:16,218 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/c578dced20324c579042babe0830bf44 2024-11-11T20:40:16,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/dd287844ddae4a61a59920427359b0a9 is 50, key is test_row_0/C:col10/1731357614512/Put/seqid=0 2024-11-11T20:40:16,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741974_1150 (size=12001) 2024-11-11T20:40:16,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357676371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357676371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357676371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357676390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357676391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,699 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/dd287844ddae4a61a59920427359b0a9 2024-11-11T20:40:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/879f6b28b4054aee88c62163acb2bb5d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/879f6b28b4054aee88c62163acb2bb5d 2024-11-11T20:40:16,722 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/879f6b28b4054aee88c62163acb2bb5d, entries=150, sequenceid=92, filesize=30.2 K 2024-11-11T20:40:16,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/c578dced20324c579042babe0830bf44 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/c578dced20324c579042babe0830bf44 2024-11-11T20:40:16,733 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/c578dced20324c579042babe0830bf44, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T20:40:16,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/dd287844ddae4a61a59920427359b0a9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dd287844ddae4a61a59920427359b0a9 2024-11-11T20:40:16,755 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dd287844ddae4a61a59920427359b0a9, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T20:40:16,757 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3c30a36b246285db6085785ae9fb5d59 in 1193ms, sequenceid=92, compaction requested=true 2024-11-11T20:40:16,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:16,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:16,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T20:40:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T20:40:16,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T20:40:16,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9060 sec 2024-11-11T20:40:16,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.9160 sec 2024-11-11T20:40:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:16,887 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:16,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T20:40:16,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e63e44a9b02d455eb8051f5e6856442d_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357616886/Put/seqid=0 2024-11-11T20:40:16,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357676915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357676915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357676917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357676926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357676916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:16,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741975_1151 (size=14594) 2024-11-11T20:40:17,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357677042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357677043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357677043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357677043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357677051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357677251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357677251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357677251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357677251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357677271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,370 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e63e44a9b02d455eb8051f5e6856442d_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e63e44a9b02d455eb8051f5e6856442d_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:17,373 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/55252b94b52143738d309fe9f4fd5d02, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:17,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/55252b94b52143738d309fe9f4fd5d02 is 175, key is test_row_0/A:col10/1731357616886/Put/seqid=0 2024-11-11T20:40:17,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741976_1152 (size=39549) 2024-11-11T20:40:17,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/55252b94b52143738d309fe9f4fd5d02 2024-11-11T20:40:17,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/2686472bef414fc58555c39d93f96491 is 50, key is test_row_0/B:col10/1731357616886/Put/seqid=0 2024-11-11T20:40:17,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741977_1153 (size=12001) 2024-11-11T20:40:17,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/2686472bef414fc58555c39d93f96491 2024-11-11T20:40:17,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/1c3f6e9cbc3644a5819f8f5df7439c40 is 50, key is test_row_0/C:col10/1731357616886/Put/seqid=0 2024-11-11T20:40:17,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741978_1154 (size=12001) 2024-11-11T20:40:17,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/1c3f6e9cbc3644a5819f8f5df7439c40 2024-11-11T20:40:17,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/55252b94b52143738d309fe9f4fd5d02 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/55252b94b52143738d309fe9f4fd5d02 2024-11-11T20:40:17,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/55252b94b52143738d309fe9f4fd5d02, entries=200, sequenceid=119, filesize=38.6 K 2024-11-11T20:40:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/2686472bef414fc58555c39d93f96491 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2686472bef414fc58555c39d93f96491 2024-11-11T20:40:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2686472bef414fc58555c39d93f96491, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T20:40:17,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/1c3f6e9cbc3644a5819f8f5df7439c40 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1c3f6e9cbc3644a5819f8f5df7439c40 2024-11-11T20:40:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1c3f6e9cbc3644a5819f8f5df7439c40, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T20:40:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 3c30a36b246285db6085785ae9fb5d59 in 607ms, sequenceid=119, compaction requested=true 2024-11-11T20:40:17,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,495 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:17,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:17,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:17,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:17,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:17,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:17,496 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,497 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:17,497 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/A is initiating minor compaction (all files) 2024-11-11T20:40:17,498 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/A in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:17,498 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b70cf6f663a94173b55d29be21d50c0c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/6f324b862f4b4db8bd69f3bf0e704c37, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/879f6b28b4054aee88c62163acb2bb5d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/55252b94b52143738d309fe9f4fd5d02] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=146.2 K 2024-11-11T20:40:17,498 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:17,498 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b70cf6f663a94173b55d29be21d50c0c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/6f324b862f4b4db8bd69f3bf0e704c37, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/879f6b28b4054aee88c62163acb2bb5d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/55252b94b52143738d309fe9f4fd5d02] 2024-11-11T20:40:17,498 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:17,498 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b70cf6f663a94173b55d29be21d50c0c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731357613659 2024-11-11T20:40:17,498 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/B is initiating minor compaction (all files) 2024-11-11T20:40:17,498 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/B in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:17,498 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b7d7a1d35fc14ca1998ab0e91615b7df, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fdb60ea61b7d4883994c3d8d0b47fa09, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/c578dced20324c579042babe0830bf44, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2686472bef414fc58555c39d93f96491] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=47.0 K 2024-11-11T20:40:17,499 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f324b862f4b4db8bd69f3bf0e704c37, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731357613835 2024-11-11T20:40:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,499 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d7a1d35fc14ca1998ab0e91615b7df, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731357613659 2024-11-11T20:40:17,499 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 879f6b28b4054aee88c62163acb2bb5d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731357614512 2024-11-11T20:40:17,499 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fdb60ea61b7d4883994c3d8d0b47fa09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731357613852 2024-11-11T20:40:17,499 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c578dced20324c579042babe0830bf44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731357614512 2024-11-11T20:40:17,499 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55252b94b52143738d309fe9f4fd5d02, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731357615709 2024-11-11T20:40:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,500 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2686472bef414fc58555c39d93f96491, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731357615709 2024-11-11T20:40:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,510 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,530 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111b7dcdae069244242b075e1a765b832fa_3c30a36b246285db6085785ae9fb5d59 store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,532 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#B#compaction#133 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,533 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/1093d37ed6714374ae3a1f6f44adabee is 50, key is test_row_0/B:col10/1731357616886/Put/seqid=0 2024-11-11T20:40:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,538 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111b7dcdae069244242b075e1a765b832fa_3c30a36b246285db6085785ae9fb5d59, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:17,539 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b7dcdae069244242b075e1a765b832fa_3c30a36b246285db6085785ae9fb5d59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741980_1156 (size=4469) 2024-11-11T20:40:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,578 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#A#compaction#132 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:17,579 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/bb239513bffa4a168b49ca67f208f838 is 175, key is test_row_0/A:col10/1731357616886/Put/seqid=0 2024-11-11T20:40:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:17,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:40:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741979_1155 (size=12241) 2024-11-11T20:40:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741981_1157 (size=31195) 2024-11-11T20:40:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111145ac8c588f6444e3ac52f1f513b49dc4_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_1/A:col10/1731357617578/Put/seqid=0 2024-11-11T20:40:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357677671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357677691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357677693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357677696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741982_1158 (size=14694) 2024-11-11T20:40:17,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357677695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:17,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,715 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111145ac8c588f6444e3ac52f1f513b49dc4_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111145ac8c588f6444e3ac52f1f513b49dc4_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:17,716 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/256252e9a35c4de0a9aeb5dec4b6d091, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:17,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/256252e9a35c4de0a9aeb5dec4b6d091 is 175, key is test_row_1/A:col10/1731357617578/Put/seqid=0 2024-11-11T20:40:17,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741983_1159 (size=39645) 2024-11-11T20:40:17,773 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/256252e9a35c4de0a9aeb5dec4b6d091 2024-11-11T20:40:17,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/3761dd7c23c9496ea70529348b63fb34 is 50, key is test_row_1/B:col10/1731357617578/Put/seqid=0 2024-11-11T20:40:17,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357677801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357677811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357677811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357677811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357677821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:17,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741984_1160 (size=9757) 2024-11-11T20:40:18,010 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/1093d37ed6714374ae3a1f6f44adabee as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1093d37ed6714374ae3a1f6f44adabee 2024-11-11T20:40:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:40:18,011 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:18,011 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T20:40:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:40:18,015 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:18,016 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:18,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:18,025 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/B of 3c30a36b246285db6085785ae9fb5d59 into 1093d37ed6714374ae3a1f6f44adabee(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:18,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,025 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:18,025 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/B, priority=12, startTime=1731357617495; duration=0sec 2024-11-11T20:40:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357678021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357678021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,025 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:18,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357678023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357678023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,026 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,026 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357678031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,032 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,032 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/C is initiating minor compaction (all files) 2024-11-11T20:40:18,032 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/C in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:18,032 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/3234a3da5587434b921550651bb0fdcb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1b0c81f9d4d1426cb664170eff0f9b91, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dd287844ddae4a61a59920427359b0a9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1c3f6e9cbc3644a5819f8f5df7439c40] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=47.0 K 2024-11-11T20:40:18,033 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3234a3da5587434b921550651bb0fdcb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731357613659 2024-11-11T20:40:18,033 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b0c81f9d4d1426cb664170eff0f9b91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731357613852 2024-11-11T20:40:18,034 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting dd287844ddae4a61a59920427359b0a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731357614512 2024-11-11T20:40:18,034 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c3f6e9cbc3644a5819f8f5df7439c40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731357615709 2024-11-11T20:40:18,050 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#C#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:18,051 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/96c773e2c1ec478895fcc53e64cff843 is 50, key is test_row_0/C:col10/1731357616886/Put/seqid=0 2024-11-11T20:40:18,068 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/bb239513bffa4a168b49ca67f208f838 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/bb239513bffa4a168b49ca67f208f838 2024-11-11T20:40:18,076 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/A of 3c30a36b246285db6085785ae9fb5d59 into bb239513bffa4a168b49ca67f208f838(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:18,077 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:18,077 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/A, priority=12, startTime=1731357617495; duration=0sec 2024-11-11T20:40:18,077 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:18,080 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:18,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741985_1161 (size=12241) 2024-11-11T20:40:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:40:18,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:40:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:18,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:18,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/3761dd7c23c9496ea70529348b63fb34 2024-11-11T20:40:18,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/a6c80ceabfb140af9a374bee4913e7e2 is 50, key is test_row_1/C:col10/1731357617578/Put/seqid=0 2024-11-11T20:40:18,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:40:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:18,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:18,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741986_1162 (size=9757) 2024-11-11T20:40:18,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/a6c80ceabfb140af9a374bee4913e7e2 2024-11-11T20:40:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:40:18,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357678331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357678331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/256252e9a35c4de0a9aeb5dec4b6d091 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/256252e9a35c4de0a9aeb5dec4b6d091 2024-11-11T20:40:18,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357678332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357678333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/256252e9a35c4de0a9aeb5dec4b6d091, entries=200, sequenceid=131, filesize=38.7 K 2024-11-11T20:40:18,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/3761dd7c23c9496ea70529348b63fb34 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3761dd7c23c9496ea70529348b63fb34 2024-11-11T20:40:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357678341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3761dd7c23c9496ea70529348b63fb34, entries=100, sequenceid=131, filesize=9.5 K 2024-11-11T20:40:18,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/a6c80ceabfb140af9a374bee4913e7e2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/a6c80ceabfb140af9a374bee4913e7e2 2024-11-11T20:40:18,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/a6c80ceabfb140af9a374bee4913e7e2, entries=100, sequenceid=131, filesize=9.5 K 2024-11-11T20:40:18,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3c30a36b246285db6085785ae9fb5d59 in 790ms, sequenceid=131, compaction requested=false 2024-11-11T20:40:18,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:18,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:40:18,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:18,482 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:40:18,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d8bc0035933246838f0aa767ba72352c_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357617633/Put/seqid=0 2024-11-11T20:40:18,517 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/96c773e2c1ec478895fcc53e64cff843 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/96c773e2c1ec478895fcc53e64cff843 2024-11-11T20:40:18,525 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/C of 3c30a36b246285db6085785ae9fb5d59 into 96c773e2c1ec478895fcc53e64cff843(size=12.0 K), total size for store is 21.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:18,525 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:18,525 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/C, priority=12, startTime=1731357617496; duration=0sec 2024-11-11T20:40:18,525 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:18,525 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:18,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741987_1163 (size=12304) 2024-11-11T20:40:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:18,550 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d8bc0035933246838f0aa767ba72352c_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d8bc0035933246838f0aa767ba72352c_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/ef9da3d86e244e479bdc846601530619, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:18,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/ef9da3d86e244e479bdc846601530619 is 175, key is test_row_0/A:col10/1731357617633/Put/seqid=0 2024-11-11T20:40:18,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741988_1164 (size=31105) 2024-11-11T20:40:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:40:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:18,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:18,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357678861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357678861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357678866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357678866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357678867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357678971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357678980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357678981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357678980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,012 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/ef9da3d86e244e479bdc846601530619 2024-11-11T20:40:19,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/cb2b29af1d4146dd9849186996e2ce32 is 50, key is test_row_0/B:col10/1731357617633/Put/seqid=0 2024-11-11T20:40:19,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741989_1165 (size=12151) 2024-11-11T20:40:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:40:19,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357679180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357679191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357679191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:19,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357679191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,468 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/cb2b29af1d4146dd9849186996e2ce32 2024-11-11T20:40:19,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/6c3c5b8fae4d41f6a7a650be176fa502 is 50, key is test_row_0/C:col10/1731357617633/Put/seqid=0 2024-11-11T20:40:19,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741990_1166 (size=12151) 2024-11-11T20:40:19,487 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/6c3c5b8fae4d41f6a7a650be176fa502 2024-11-11T20:40:19,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357679491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357679501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:19,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357679501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:19,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357679501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/ef9da3d86e244e479bdc846601530619 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ef9da3d86e244e479bdc846601530619 2024-11-11T20:40:19,516 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ef9da3d86e244e479bdc846601530619, entries=150, sequenceid=157, filesize=30.4 K 2024-11-11T20:40:19,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/cb2b29af1d4146dd9849186996e2ce32 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/cb2b29af1d4146dd9849186996e2ce32 2024-11-11T20:40:19,528 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/cb2b29af1d4146dd9849186996e2ce32, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T20:40:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/6c3c5b8fae4d41f6a7a650be176fa502 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6c3c5b8fae4d41f6a7a650be176fa502 2024-11-11T20:40:19,534 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6c3c5b8fae4d41f6a7a650be176fa502, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T20:40:19,536 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 3c30a36b246285db6085785ae9fb5d59 in 1053ms, sequenceid=157, compaction requested=true 2024-11-11T20:40:19,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:19,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:19,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T20:40:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T20:40:19,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T20:40:19,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5210 sec 2024-11-11T20:40:19,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5290 sec 2024-11-11T20:40:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:19,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:19,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T20:40:19,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e11ad90610ff4999b247f1cd9b264edb_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357618863/Put/seqid=0 2024-11-11T20:40:19,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741991_1167 (size=14794) 2024-11-11T20:40:19,887 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:19,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e11ad90610ff4999b247f1cd9b264edb_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e11ad90610ff4999b247f1cd9b264edb_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:19,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/f8fa505f9f1d40dab1a067792d4b0a66, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:19,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/f8fa505f9f1d40dab1a067792d4b0a66 is 175, key is test_row_0/A:col10/1731357618863/Put/seqid=0 2024-11-11T20:40:19,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741992_1168 (size=39749) 2024-11-11T20:40:19,919 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/f8fa505f9f1d40dab1a067792d4b0a66 2024-11-11T20:40:19,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/ddbf54c1b5fe462f86ec414355c18344 is 50, key is test_row_0/B:col10/1731357618863/Put/seqid=0 2024-11-11T20:40:19,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741993_1169 (size=12151) 2024-11-11T20:40:19,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/ddbf54c1b5fe462f86ec414355c18344 2024-11-11T20:40:19,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/d2b3f2a307e344418f668de370e45b46 is 50, key is test_row_0/C:col10/1731357618863/Put/seqid=0 2024-11-11T20:40:19,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357679983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:19,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741994_1170 (size=12151) 2024-11-11T20:40:20,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357680001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:20,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357680011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357680013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:20,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357680021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:20,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357680091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:40:20,160 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:20,161 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T20:40:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:40:20,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:20,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:20,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:40:20,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357680300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:20,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T20:40:20,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:20,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:20,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:20,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:20,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:20,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/d2b3f2a307e344418f668de370e45b46 2024-11-11T20:40:20,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/f8fa505f9f1d40dab1a067792d4b0a66 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f8fa505f9f1d40dab1a067792d4b0a66 2024-11-11T20:40:20,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f8fa505f9f1d40dab1a067792d4b0a66, entries=200, sequenceid=172, filesize=38.8 K 2024-11-11T20:40:20,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/ddbf54c1b5fe462f86ec414355c18344 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ddbf54c1b5fe462f86ec414355c18344 2024-11-11T20:40:20,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ddbf54c1b5fe462f86ec414355c18344, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T20:40:20,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/d2b3f2a307e344418f668de370e45b46 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d2b3f2a307e344418f668de370e45b46 2024-11-11T20:40:20,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d2b3f2a307e344418f668de370e45b46, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T20:40:20,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3c30a36b246285db6085785ae9fb5d59 in 570ms, sequenceid=172, compaction requested=true 2024-11-11T20:40:20,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:20,445 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:20,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:20,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:20,445 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:20,447 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:20,447 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/B is initiating minor compaction (all files) 2024-11-11T20:40:20,447 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/B in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:20,447 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1093d37ed6714374ae3a1f6f44adabee, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3761dd7c23c9496ea70529348b63fb34, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/cb2b29af1d4146dd9849186996e2ce32, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ddbf54c1b5fe462f86ec414355c18344] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=45.2 K 2024-11-11T20:40:20,448 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:20,448 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/A is initiating minor compaction (all files) 2024-11-11T20:40:20,448 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/A in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:20,448 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/bb239513bffa4a168b49ca67f208f838, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/256252e9a35c4de0a9aeb5dec4b6d091, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ef9da3d86e244e479bdc846601530619, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f8fa505f9f1d40dab1a067792d4b0a66] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=138.4 K 2024-11-11T20:40:20,448 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:20,448 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/bb239513bffa4a168b49ca67f208f838, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/256252e9a35c4de0a9aeb5dec4b6d091, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ef9da3d86e244e479bdc846601530619, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f8fa505f9f1d40dab1a067792d4b0a66] 2024-11-11T20:40:20,449 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1093d37ed6714374ae3a1f6f44adabee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731357615709 2024-11-11T20:40:20,449 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb239513bffa4a168b49ca67f208f838, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731357615709 2024-11-11T20:40:20,450 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3761dd7c23c9496ea70529348b63fb34, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731357617576 2024-11-11T20:40:20,450 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 256252e9a35c4de0a9aeb5dec4b6d091, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731357616908 2024-11-11T20:40:20,450 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2b29af1d4146dd9849186996e2ce32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731357617633 2024-11-11T20:40:20,451 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef9da3d86e244e479bdc846601530619, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731357617633 2024-11-11T20:40:20,451 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ddbf54c1b5fe462f86ec414355c18344, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731357618863 2024-11-11T20:40:20,451 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8fa505f9f1d40dab1a067792d4b0a66, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731357618863 2024-11-11T20:40:20,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T20:40:20,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:20,471 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:40:20,471 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:20,476 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:20,476 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111128d23375d5ee4757b93bc78b8ceaae7d_3c30a36b246285db6085785ae9fb5d59 store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:20,477 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/9c53d3fb1eff45f9bf755f9c0fab9a76 is 50, key is test_row_0/B:col10/1731357618863/Put/seqid=0 2024-11-11T20:40:20,480 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111128d23375d5ee4757b93bc78b8ceaae7d_3c30a36b246285db6085785ae9fb5d59, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:20,480 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111128d23375d5ee4757b93bc78b8ceaae7d_3c30a36b246285db6085785ae9fb5d59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:40:20,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113720f787da984925a3576761afd5a7a9_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357619971/Put/seqid=0 2024-11-11T20:40:20,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741996_1172 (size=4469) 2024-11-11T20:40:20,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741995_1171 (size=12527) 2024-11-11T20:40:20,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741997_1173 (size=12304) 2024-11-11T20:40:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:20,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357680717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:40:20,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357680831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:20,989 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#A#compaction#144 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:20,989 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/22d7dde9161640e7972db1274f9371a9 is 175, key is test_row_0/A:col10/1731357618863/Put/seqid=0 2024-11-11T20:40:20,997 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/9c53d3fb1eff45f9bf755f9c0fab9a76 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9c53d3fb1eff45f9bf755f9c0fab9a76 2024-11-11T20:40:21,003 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/B of 3c30a36b246285db6085785ae9fb5d59 into 9c53d3fb1eff45f9bf755f9c0fab9a76(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:21,003 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:21,003 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/B, priority=12, startTime=1731357620445; duration=0sec 2024-11-11T20:40:21,003 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:21,003 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:21,003 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:21,005 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:21,005 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/C is initiating minor compaction (all files) 2024-11-11T20:40:21,005 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/C in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,006 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/96c773e2c1ec478895fcc53e64cff843, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/a6c80ceabfb140af9a374bee4913e7e2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6c3c5b8fae4d41f6a7a650be176fa502, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d2b3f2a307e344418f668de370e45b46] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=45.2 K 2024-11-11T20:40:21,007 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c773e2c1ec478895fcc53e64cff843, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731357615709 2024-11-11T20:40:21,007 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c80ceabfb140af9a374bee4913e7e2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731357617576 2024-11-11T20:40:21,008 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c3c5b8fae4d41f6a7a650be176fa502, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731357617633 2024-11-11T20:40:21,008 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d2b3f2a307e344418f668de370e45b46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731357618863 2024-11-11T20:40:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:21,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:21,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357681020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357681020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:21,022 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_0' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:21,022 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_0' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357681025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:21,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357681031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:21,064 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_0' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:21,065 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:21,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357681064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:21,073 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113720f787da984925a3576761afd5a7a9_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113720f787da984925a3576761afd5a7a9_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:21,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/e985ce1e00424288a255bcc621289b75, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:21,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741998_1174 (size=31481) 2024-11-11T20:40:21,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/e985ce1e00424288a255bcc621289b75 is 175, key is test_row_0/A:col10/1731357619971/Put/seqid=0 2024-11-11T20:40:21,085 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:21,086 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/73b1254f2d0f4f7180780ca9e7356030 is 50, key is test_row_0/C:col10/1731357618863/Put/seqid=0 2024-11-11T20:40:21,097 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/22d7dde9161640e7972db1274f9371a9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22d7dde9161640e7972db1274f9371a9 2024-11-11T20:40:21,104 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/A of 3c30a36b246285db6085785ae9fb5d59 into 22d7dde9161640e7972db1274f9371a9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:21,104 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:21,104 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/A, priority=12, startTime=1731357620445; duration=0sec 2024-11-11T20:40:21,104 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:21,104 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:21,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741999_1175 (size=31105) 2024-11-11T20:40:21,118 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/e985ce1e00424288a255bcc621289b75 2024-11-11T20:40:21,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742000_1176 (size=12527) 2024-11-11T20:40:21,143 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/73b1254f2d0f4f7180780ca9e7356030 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/73b1254f2d0f4f7180780ca9e7356030 2024-11-11T20:40:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/f4a25a80ffdb45b387dc0598ac488093 is 50, key is test_row_0/B:col10/1731357619971/Put/seqid=0 2024-11-11T20:40:21,151 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/C of 3c30a36b246285db6085785ae9fb5d59 into 73b1254f2d0f4f7180780ca9e7356030(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:21,151 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:21,151 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/C, priority=12, startTime=1731357620446; duration=0sec 2024-11-11T20:40:21,152 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:21,152 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:21,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742001_1177 (size=12151) 2024-11-11T20:40:21,183 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/f4a25a80ffdb45b387dc0598ac488093 2024-11-11T20:40:21,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/c58c2927167e4307b177167b85cfed08 is 50, key is test_row_0/C:col10/1731357619971/Put/seqid=0 2024-11-11T20:40:21,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742002_1178 (size=12151) 2024-11-11T20:40:21,241 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/c58c2927167e4307b177167b85cfed08 2024-11-11T20:40:21,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/e985ce1e00424288a255bcc621289b75 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e985ce1e00424288a255bcc621289b75 2024-11-11T20:40:21,267 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e985ce1e00424288a255bcc621289b75, entries=150, sequenceid=194, filesize=30.4 K 2024-11-11T20:40:21,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/f4a25a80ffdb45b387dc0598ac488093 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f4a25a80ffdb45b387dc0598ac488093 2024-11-11T20:40:21,274 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f4a25a80ffdb45b387dc0598ac488093, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T20:40:21,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/c58c2927167e4307b177167b85cfed08 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c58c2927167e4307b177167b85cfed08 2024-11-11T20:40:21,280 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c58c2927167e4307b177167b85cfed08, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T20:40:21,282 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3c30a36b246285db6085785ae9fb5d59 in 810ms, sequenceid=194, compaction requested=false 2024-11-11T20:40:21,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:21,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T20:40:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T20:40:21,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T20:40:21,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1200 sec 2024-11-11T20:40:21,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1260 sec 2024-11-11T20:40:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:40:21,301 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:21,302 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T20:40:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T20:40:21,304 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:21,305 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:21,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:21,385 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:21,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T20:40:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:21,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111164659b74a06b48b49ce97b03ec488c22_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357621383/Put/seqid=0 2024-11-11T20:40:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T20:40:21,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742003_1179 (size=12304) 2024-11-11T20:40:21,428 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:21,433 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111164659b74a06b48b49ce97b03ec488c22_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111164659b74a06b48b49ce97b03ec488c22_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:21,435 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/4a38d0cbab334e3c81c37401e8116d51, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:21,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/4a38d0cbab334e3c81c37401e8116d51 is 175, key is test_row_0/A:col10/1731357621383/Put/seqid=0 2024-11-11T20:40:21,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T20:40:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:21,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:21,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:21,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742004_1180 (size=31105) 2024-11-11T20:40:21,468 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/4a38d0cbab334e3c81c37401e8116d51 2024-11-11T20:40:21,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/b92af0662c05486fb6818ca268ababd8 is 50, key is test_row_0/B:col10/1731357621383/Put/seqid=0 2024-11-11T20:40:21,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357681525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:21,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742005_1181 (size=12151) 2024-11-11T20:40:21,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/b92af0662c05486fb6818ca268ababd8 2024-11-11T20:40:21,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e155fe1a6356456e90aea26cd8f7aeb2 is 50, key is test_row_0/C:col10/1731357621383/Put/seqid=0 2024-11-11T20:40:21,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T20:40:21,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:21,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:21,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T20:40:21,632 INFO [master/51ca66f7ee3c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T20:40:21,632 INFO [master/51ca66f7ee3c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T20:40:21,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357681631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:21,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742006_1182 (size=12151) 2024-11-11T20:40:21,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e155fe1a6356456e90aea26cd8f7aeb2 2024-11-11T20:40:21,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/4a38d0cbab334e3c81c37401e8116d51 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/4a38d0cbab334e3c81c37401e8116d51 2024-11-11T20:40:21,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/4a38d0cbab334e3c81c37401e8116d51, entries=150, sequenceid=212, filesize=30.4 K 2024-11-11T20:40:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/b92af0662c05486fb6818ca268ababd8 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b92af0662c05486fb6818ca268ababd8 2024-11-11T20:40:21,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b92af0662c05486fb6818ca268ababd8, entries=150, sequenceid=212, filesize=11.9 K 2024-11-11T20:40:21,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e155fe1a6356456e90aea26cd8f7aeb2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e155fe1a6356456e90aea26cd8f7aeb2 2024-11-11T20:40:21,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e155fe1a6356456e90aea26cd8f7aeb2, entries=150, sequenceid=212, filesize=11.9 K 2024-11-11T20:40:21,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3c30a36b246285db6085785ae9fb5d59 in 305ms, sequenceid=212, compaction requested=true 2024-11-11T20:40:21,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:21,690 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:21,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:21,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:21,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:21,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:21,690 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:21,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:21,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:21,691 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:21,691 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/A is initiating minor compaction (all files) 2024-11-11T20:40:21,691 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/A in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,692 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22d7dde9161640e7972db1274f9371a9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e985ce1e00424288a255bcc621289b75, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/4a38d0cbab334e3c81c37401e8116d51] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=91.5 K 2024-11-11T20:40:21,692 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,692 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22d7dde9161640e7972db1274f9371a9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e985ce1e00424288a255bcc621289b75, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/4a38d0cbab334e3c81c37401e8116d51] 2024-11-11T20:40:21,692 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:21,692 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/B is initiating minor compaction (all files) 2024-11-11T20:40:21,692 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/B in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,692 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9c53d3fb1eff45f9bf755f9c0fab9a76, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f4a25a80ffdb45b387dc0598ac488093, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b92af0662c05486fb6818ca268ababd8] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=36.0 K 2024-11-11T20:40:21,692 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22d7dde9161640e7972db1274f9371a9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731357618863 2024-11-11T20:40:21,693 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c53d3fb1eff45f9bf755f9c0fab9a76, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731357618863 2024-11-11T20:40:21,696 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f4a25a80ffdb45b387dc0598ac488093, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731357619963 2024-11-11T20:40:21,698 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e985ce1e00424288a255bcc621289b75, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731357619963 2024-11-11T20:40:21,699 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b92af0662c05486fb6818ca268ababd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731357620706 2024-11-11T20:40:21,699 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a38d0cbab334e3c81c37401e8116d51, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731357620706 2024-11-11T20:40:21,709 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:21,711 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:21,712 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/22c49ba7d58145549bfce5866b9c8f0a is 50, key is test_row_0/B:col10/1731357621383/Put/seqid=0 2024-11-11T20:40:21,722 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110e887a201e5a438f834f00c2dcfed8cf_3c30a36b246285db6085785ae9fb5d59 store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:21,726 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110e887a201e5a438f834f00c2dcfed8cf_3c30a36b246285db6085785ae9fb5d59, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:21,726 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110e887a201e5a438f834f00c2dcfed8cf_3c30a36b246285db6085785ae9fb5d59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:21,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742008_1184 (size=12629) 2024-11-11T20:40:21,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T20:40:21,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:21,772 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T20:40:21,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742007_1183 (size=4469) 2024-11-11T20:40:21,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ac19297a642f4ac18d56d04f7428fccc_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357621520/Put/seqid=0 2024-11-11T20:40:21,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742009_1185 (size=12304) 2024-11-11T20:40:21,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:21,828 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ac19297a642f4ac18d56d04f7428fccc_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ac19297a642f4ac18d56d04f7428fccc_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:21,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/a76f60ea073242ec96cc6b27ab8b5ce5, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:21,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/a76f60ea073242ec96cc6b27ab8b5ce5 is 175, key is test_row_0/A:col10/1731357621520/Put/seqid=0 2024-11-11T20:40:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:21,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:21,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742010_1186 (size=31105) 2024-11-11T20:40:21,880 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/a76f60ea073242ec96cc6b27ab8b5ce5 2024-11-11T20:40:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/1816ddf5409e47f0a040c8b83c032b21 is 50, key is test_row_0/B:col10/1731357621520/Put/seqid=0 2024-11-11T20:40:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357681916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T20:40:21,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742011_1187 (size=12151) 2024-11-11T20:40:21,951 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/1816ddf5409e47f0a040c8b83c032b21 2024-11-11T20:40:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/7977e3d818c84cbeabf88967970c3267 is 50, key is test_row_0/C:col10/1731357621520/Put/seqid=0 2024-11-11T20:40:21,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742012_1188 (size=12151) 2024-11-11T20:40:21,981 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/7977e3d818c84cbeabf88967970c3267 2024-11-11T20:40:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/a76f60ea073242ec96cc6b27ab8b5ce5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a76f60ea073242ec96cc6b27ab8b5ce5 2024-11-11T20:40:22,001 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a76f60ea073242ec96cc6b27ab8b5ce5, entries=150, sequenceid=233, filesize=30.4 K 2024-11-11T20:40:22,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/1816ddf5409e47f0a040c8b83c032b21 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1816ddf5409e47f0a040c8b83c032b21 2024-11-11T20:40:22,012 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1816ddf5409e47f0a040c8b83c032b21, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T20:40:22,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/7977e3d818c84cbeabf88967970c3267 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7977e3d818c84cbeabf88967970c3267 2024-11-11T20:40:22,023 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7977e3d818c84cbeabf88967970c3267, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T20:40:22,025 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3c30a36b246285db6085785ae9fb5d59 in 253ms, sequenceid=233, compaction requested=true 2024-11-11T20:40:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T20:40:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T20:40:22,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T20:40:22,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 722 msec 2024-11-11T20:40:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:22,034 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:22,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T20:40:22,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 728 msec 2024-11-11T20:40:22,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112b486ee4cfb74c51a229f556e3fc5fd5_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357622032/Put/seqid=0 2024-11-11T20:40:22,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742013_1189 (size=14794) 2024-11-11T20:40:22,051 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:22,056 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112b486ee4cfb74c51a229f556e3fc5fd5_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b486ee4cfb74c51a229f556e3fc5fd5_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:22,058 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/357a5a137ee2478cbaa889929aaf665d, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:22,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/357a5a137ee2478cbaa889929aaf665d is 175, key is test_row_0/A:col10/1731357622032/Put/seqid=0 2024-11-11T20:40:22,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742014_1190 (size=39749) 2024-11-11T20:40:22,066 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/357a5a137ee2478cbaa889929aaf665d 2024-11-11T20:40:22,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/2d705324ee554807944a71379794d645 is 50, key is test_row_0/B:col10/1731357622032/Put/seqid=0 2024-11-11T20:40:22,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742015_1191 (size=12151) 2024-11-11T20:40:22,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/2d705324ee554807944a71379794d645 2024-11-11T20:40:22,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357682140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:22,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/d3351ffa32fb4c5cb7eb8fbe42bd22ae is 50, key is test_row_0/C:col10/1731357622032/Put/seqid=0 2024-11-11T20:40:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:22,172 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/22c49ba7d58145549bfce5866b9c8f0a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/22c49ba7d58145549bfce5866b9c8f0a 2024-11-11T20:40:22,178 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/B of 3c30a36b246285db6085785ae9fb5d59 into 22c49ba7d58145549bfce5866b9c8f0a(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:22,178 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:22,178 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/B, priority=13, startTime=1731357621690; duration=0sec 2024-11-11T20:40:22,178 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:22,179 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:22,179 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:22,179 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#A#compaction#153 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:22,179 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/34708b4f47a640d09d34c9e3642183cd is 175, key is test_row_0/A:col10/1731357621383/Put/seqid=0 2024-11-11T20:40:22,180 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:22,180 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/C is initiating minor compaction (all files) 2024-11-11T20:40:22,180 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/C in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,181 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/73b1254f2d0f4f7180780ca9e7356030, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c58c2927167e4307b177167b85cfed08, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e155fe1a6356456e90aea26cd8f7aeb2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7977e3d818c84cbeabf88967970c3267] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=47.8 K 2024-11-11T20:40:22,182 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b1254f2d0f4f7180780ca9e7356030, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731357618863 2024-11-11T20:40:22,183 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c58c2927167e4307b177167b85cfed08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731357619963 2024-11-11T20:40:22,184 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting e155fe1a6356456e90aea26cd8f7aeb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731357620706 2024-11-11T20:40:22,184 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7977e3d818c84cbeabf88967970c3267, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731357621472 2024-11-11T20:40:22,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742016_1192 (size=12151) 2024-11-11T20:40:22,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/d3351ffa32fb4c5cb7eb8fbe42bd22ae 2024-11-11T20:40:22,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/357a5a137ee2478cbaa889929aaf665d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/357a5a137ee2478cbaa889929aaf665d 2024-11-11T20:40:22,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/357a5a137ee2478cbaa889929aaf665d, entries=200, sequenceid=249, filesize=38.8 K 2024-11-11T20:40:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/2d705324ee554807944a71379794d645 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2d705324ee554807944a71379794d645 2024-11-11T20:40:22,221 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:22,221 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e3fbc185a44e403197013c50fe020cc6 is 50, key is test_row_0/C:col10/1731357621520/Put/seqid=0 2024-11-11T20:40:22,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742017_1193 (size=31583) 2024-11-11T20:40:22,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742018_1194 (size=12663) 2024-11-11T20:40:22,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2d705324ee554807944a71379794d645, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T20:40:22,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/d3351ffa32fb4c5cb7eb8fbe42bd22ae as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d3351ffa32fb4c5cb7eb8fbe42bd22ae 2024-11-11T20:40:22,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d3351ffa32fb4c5cb7eb8fbe42bd22ae, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T20:40:22,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3c30a36b246285db6085785ae9fb5d59 in 211ms, sequenceid=249, compaction requested=true 2024-11-11T20:40:22,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:22,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:22,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:22,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:22,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T20:40:22,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:22,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T20:40:22,246 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/34708b4f47a640d09d34c9e3642183cd as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/34708b4f47a640d09d34c9e3642183cd 2024-11-11T20:40:22,252 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/A of 3c30a36b246285db6085785ae9fb5d59 into 34708b4f47a640d09d34c9e3642183cd(size=30.8 K), total size for store is 100.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:22,253 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:22,253 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/A, priority=13, startTime=1731357621690; duration=0sec 2024-11-11T20:40:22,253 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T20:40:22,253 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:22,253 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:22,253 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-11T20:40:22,254 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T20:40:22,254 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T20:40:22,254 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. because compaction request was cancelled 2024-11-11T20:40:22,254 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:22,254 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:22,256 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:22,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T20:40:22,256 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:22,256 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/A is initiating minor compaction (all files) 2024-11-11T20:40:22,257 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/A in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,257 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/34708b4f47a640d09d34c9e3642183cd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a76f60ea073242ec96cc6b27ab8b5ce5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/357a5a137ee2478cbaa889929aaf665d] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=100.0 K 2024-11-11T20:40:22,257 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,257 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/34708b4f47a640d09d34c9e3642183cd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a76f60ea073242ec96cc6b27ab8b5ce5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/357a5a137ee2478cbaa889929aaf665d] 2024-11-11T20:40:22,258 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34708b4f47a640d09d34c9e3642183cd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731357620706 2024-11-11T20:40:22,258 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76f60ea073242ec96cc6b27ab8b5ce5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731357621472 2024-11-11T20:40:22,259 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 357a5a137ee2478cbaa889929aaf665d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731357621899 2024-11-11T20:40:22,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112e810e03dee344dd8c6d1ad8b87c29f6_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357622138/Put/seqid=0 2024-11-11T20:40:22,270 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:22,286 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115f2767f311714aac83582ec2be03ba61_3c30a36b246285db6085785ae9fb5d59 store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:22,288 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115f2767f311714aac83582ec2be03ba61_3c30a36b246285db6085785ae9fb5d59, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:22,288 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f2767f311714aac83582ec2be03ba61_3c30a36b246285db6085785ae9fb5d59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:22,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742019_1195 (size=14994) 2024-11-11T20:40:22,303 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:22,307 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112e810e03dee344dd8c6d1ad8b87c29f6_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112e810e03dee344dd8c6d1ad8b87c29f6_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:22,309 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/840e30c2da9a4b659afbd67c462ebe7a, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:22,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/840e30c2da9a4b659afbd67c462ebe7a is 175, key is test_row_0/A:col10/1731357622138/Put/seqid=0 2024-11-11T20:40:22,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742020_1196 (size=4469) 2024-11-11T20:40:22,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742021_1197 (size=39949) 2024-11-11T20:40:22,324 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#A#compaction#163 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:22,325 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/840e30c2da9a4b659afbd67c462ebe7a 2024-11-11T20:40:22,325 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/d5cfad00901148338d98bcda0238b85e is 175, key is test_row_0/A:col10/1731357622032/Put/seqid=0 2024-11-11T20:40:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357682316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:22,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742022_1198 (size=31685) 2024-11-11T20:40:22,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/a08d4f4e8988458095ad7ae18ae56626 is 50, key is test_row_0/B:col10/1731357622138/Put/seqid=0 2024-11-11T20:40:22,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742023_1199 (size=12301) 2024-11-11T20:40:22,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357682430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T20:40:22,441 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:22,441 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T20:40:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T20:40:22,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:22,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:22,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T20:40:22,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T20:40:22,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:22,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:22,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:22,644 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e3fbc185a44e403197013c50fe020cc6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e3fbc185a44e403197013c50fe020cc6 2024-11-11T20:40:22,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357682641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:22,651 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/C of 3c30a36b246285db6085785ae9fb5d59 into e3fbc185a44e403197013c50fe020cc6(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:22,651 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:22,651 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/C, priority=12, startTime=1731357621690; duration=0sec 2024-11-11T20:40:22,651 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:22,651 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:22,651 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:22,653 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:22,653 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/B is initiating minor compaction (all files) 2024-11-11T20:40:22,653 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/B in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,653 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/22c49ba7d58145549bfce5866b9c8f0a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1816ddf5409e47f0a040c8b83c032b21, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2d705324ee554807944a71379794d645] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=36.1 K 2024-11-11T20:40:22,653 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c49ba7d58145549bfce5866b9c8f0a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731357620706 2024-11-11T20:40:22,654 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1816ddf5409e47f0a040c8b83c032b21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731357621472 2024-11-11T20:40:22,654 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d705324ee554807944a71379794d645, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731357621899 2024-11-11T20:40:22,665 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:22,665 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/4ecd39684fe84e7e8a599dec1cb0268d is 50, key is test_row_0/B:col10/1731357622032/Put/seqid=0 2024-11-11T20:40:22,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742024_1200 (size=12731) 2024-11-11T20:40:22,748 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/d5cfad00901148338d98bcda0238b85e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/d5cfad00901148338d98bcda0238b85e 2024-11-11T20:40:22,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T20:40:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:22,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/a08d4f4e8988458095ad7ae18ae56626 2024-11-11T20:40:22,757 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/A of 3c30a36b246285db6085785ae9fb5d59 into d5cfad00901148338d98bcda0238b85e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:22,757 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:22,757 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/A, priority=13, startTime=1731357622245; duration=0sec 2024-11-11T20:40:22,757 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:22,758 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T20:40:22,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/04d7de26407746c88a8132171e2ffeca is 50, key is test_row_0/C:col10/1731357622138/Put/seqid=0 2024-11-11T20:40:22,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742025_1201 (size=12301) 2024-11-11T20:40:22,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T20:40:22,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:22,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:22,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:22,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:22,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357682950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:23,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T20:40:23,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:23,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T20:40:23,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357683072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:23,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357683072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:23,075 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:23,077 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_0' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:23,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357683081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:23,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357683082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:23,085 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_0' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:23,085 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_0' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:23,105 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/4ecd39684fe84e7e8a599dec1cb0268d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4ecd39684fe84e7e8a599dec1cb0268d 2024-11-11T20:40:23,114 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/B of 3c30a36b246285db6085785ae9fb5d59 into 4ecd39684fe84e7e8a599dec1cb0268d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:23,114 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:23,115 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/B, priority=13, startTime=1731357622245; duration=0sec 2024-11-11T20:40:23,115 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:23,115 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:23,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/04d7de26407746c88a8132171e2ffeca 2024-11-11T20:40:23,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/840e30c2da9a4b659afbd67c462ebe7a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/840e30c2da9a4b659afbd67c462ebe7a 2024-11-11T20:40:23,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/840e30c2da9a4b659afbd67c462ebe7a, entries=200, sequenceid=272, filesize=39.0 K 2024-11-11T20:40:23,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/a08d4f4e8988458095ad7ae18ae56626 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a08d4f4e8988458095ad7ae18ae56626 2024-11-11T20:40:23,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a08d4f4e8988458095ad7ae18ae56626, entries=150, sequenceid=272, filesize=12.0 K 2024-11-11T20:40:23,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/04d7de26407746c88a8132171e2ffeca as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/04d7de26407746c88a8132171e2ffeca 2024-11-11T20:40:23,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T20:40:23,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:23,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/04d7de26407746c88a8132171e2ffeca, entries=150, sequenceid=272, filesize=12.0 K 2024-11-11T20:40:23,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3c30a36b246285db6085785ae9fb5d59 in 958ms, sequenceid=272, compaction requested=true 2024-11-11T20:40:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:23,215 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T20:40:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:23,215 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T20:40:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T20:40:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T20:40:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. because compaction request was cancelled 2024-11-11T20:40:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T20:40:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T20:40:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. because compaction request was cancelled 2024-11-11T20:40:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:23,217 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:23,217 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/C is initiating minor compaction (all files) 2024-11-11T20:40:23,217 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/C in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,217 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e3fbc185a44e403197013c50fe020cc6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d3351ffa32fb4c5cb7eb8fbe42bd22ae, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/04d7de26407746c88a8132171e2ffeca] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=36.2 K 2024-11-11T20:40:23,218 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3fbc185a44e403197013c50fe020cc6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731357621472 2024-11-11T20:40:23,218 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3351ffa32fb4c5cb7eb8fbe42bd22ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731357621899 2024-11-11T20:40:23,218 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04d7de26407746c88a8132171e2ffeca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357622104 2024-11-11T20:40:23,224 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:23,225 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/dfb53b677c3c43548d3767786d2ee5e0 is 50, key is test_row_0/C:col10/1731357622138/Put/seqid=0 2024-11-11T20:40:23,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742026_1202 (size=12915) 2024-11-11T20:40:23,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T20:40:23,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,364 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T20:40:23,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112f71c4d4a5e5477980c4377cab76cf77_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357622313/Put/seqid=0 2024-11-11T20:40:23,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742027_1203 (size=12454) 2024-11-11T20:40:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:23,403 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112f71c4d4a5e5477980c4377cab76cf77_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112f71c4d4a5e5477980c4377cab76cf77_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/22002402b612457787c51b3da75cbae7, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:23,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/22002402b612457787c51b3da75cbae7 is 175, key is test_row_0/A:col10/1731357622313/Put/seqid=0 2024-11-11T20:40:23,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742028_1204 (size=31255) 2024-11-11T20:40:23,422 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/22002402b612457787c51b3da75cbae7 2024-11-11T20:40:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/5a8c705c9570425dab468c58f4178952 is 50, key is test_row_0/B:col10/1731357622313/Put/seqid=0 2024-11-11T20:40:23,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742029_1205 (size=12301) 2024-11-11T20:40:23,445 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/5a8c705c9570425dab468c58f4178952 2024-11-11T20:40:23,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:23,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/7d29d93069234baabdc4c2f3e6367334 is 50, key is test_row_0/C:col10/1731357622313/Put/seqid=0 2024-11-11T20:40:23,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742030_1206 (size=12301) 2024-11-11T20:40:23,487 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/7d29d93069234baabdc4c2f3e6367334 2024-11-11T20:40:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/22002402b612457787c51b3da75cbae7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22002402b612457787c51b3da75cbae7 2024-11-11T20:40:23,504 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22002402b612457787c51b3da75cbae7, entries=150, sequenceid=290, filesize=30.5 K 2024-11-11T20:40:23,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/5a8c705c9570425dab468c58f4178952 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/5a8c705c9570425dab468c58f4178952 2024-11-11T20:40:23,516 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/5a8c705c9570425dab468c58f4178952, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T20:40:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/7d29d93069234baabdc4c2f3e6367334 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7d29d93069234baabdc4c2f3e6367334 2024-11-11T20:40:23,524 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7d29d93069234baabdc4c2f3e6367334, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T20:40:23,525 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=80.51 KB/82440 for 3c30a36b246285db6085785ae9fb5d59 in 161ms, sequenceid=290, compaction requested=true 2024-11-11T20:40:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T20:40:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:23,527 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T20:40:23,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T20:40:23,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T20:40:23,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0860 sec 2024-11-11T20:40:23,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fc953f5e1d8c49dda69a7df33a601cbb_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357623482/Put/seqid=0 2024-11-11T20:40:23,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.0930 sec 2024-11-11T20:40:23,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742031_1207 (size=14994) 2024-11-11T20:40:23,558 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:23,562 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fc953f5e1d8c49dda69a7df33a601cbb_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fc953f5e1d8c49dda69a7df33a601cbb_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:23,563 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/ab15c9b655504d19b36bdeeaa1c22406, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:23,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/ab15c9b655504d19b36bdeeaa1c22406 is 175, key is test_row_0/A:col10/1731357623482/Put/seqid=0 2024-11-11T20:40:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T20:40:23,581 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:23,581 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:23,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742032_1208 (size=39949) 2024-11-11T20:40:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-11T20:40:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T20:40:23,585 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:23,586 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:23,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:23,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357683615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:23,634 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/dfb53b677c3c43548d3767786d2ee5e0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dfb53b677c3c43548d3767786d2ee5e0 2024-11-11T20:40:23,642 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/C of 3c30a36b246285db6085785ae9fb5d59 into dfb53b677c3c43548d3767786d2ee5e0(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:23,642 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:23,642 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/C, priority=13, startTime=1731357623216; duration=0sec 2024-11-11T20:40:23,642 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:23,642 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T20:40:23,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357683731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:23,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T20:40:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T20:40:23,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:23,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:23,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T20:40:23,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357683940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:23,983 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/ab15c9b655504d19b36bdeeaa1c22406 2024-11-11T20:40:23,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/9e80cb4f59364083bb11039d16779a82 is 50, key is test_row_0/B:col10/1731357623482/Put/seqid=0 2024-11-11T20:40:23,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742033_1209 (size=12301) 2024-11-11T20:40:23,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/9e80cb4f59364083bb11039d16779a82 2024-11-11T20:40:24,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/830adb496c1d4730afda09ae061dc0c4 is 50, key is test_row_0/C:col10/1731357623482/Put/seqid=0 2024-11-11T20:40:24,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742034_1210 (size=12301) 2024-11-11T20:40:24,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/830adb496c1d4730afda09ae061dc0c4 2024-11-11T20:40:24,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/ab15c9b655504d19b36bdeeaa1c22406 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ab15c9b655504d19b36bdeeaa1c22406 2024-11-11T20:40:24,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T20:40:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:24,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:24,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ab15c9b655504d19b36bdeeaa1c22406, entries=200, sequenceid=306, filesize=39.0 K 2024-11-11T20:40:24,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/9e80cb4f59364083bb11039d16779a82 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9e80cb4f59364083bb11039d16779a82 2024-11-11T20:40:24,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9e80cb4f59364083bb11039d16779a82, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T20:40:24,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/830adb496c1d4730afda09ae061dc0c4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/830adb496c1d4730afda09ae061dc0c4 2024-11-11T20:40:24,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/830adb496c1d4730afda09ae061dc0c4, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T20:40:24,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3c30a36b246285db6085785ae9fb5d59 in 541ms, sequenceid=306, compaction requested=true 2024-11-11T20:40:24,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:24,069 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:24,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:24,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:24,071 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:24,071 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142838 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:24,071 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/A is initiating minor compaction (all files) 2024-11-11T20:40:24,071 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/A in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:24,072 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/d5cfad00901148338d98bcda0238b85e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/840e30c2da9a4b659afbd67c462ebe7a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22002402b612457787c51b3da75cbae7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ab15c9b655504d19b36bdeeaa1c22406] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=139.5 K 2024-11-11T20:40:24,072 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:24,072 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/d5cfad00901148338d98bcda0238b85e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/840e30c2da9a4b659afbd67c462ebe7a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22002402b612457787c51b3da75cbae7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ab15c9b655504d19b36bdeeaa1c22406] 2024-11-11T20:40:24,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:24,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:24,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:24,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:24,074 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5cfad00901148338d98bcda0238b85e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731357621899 2024-11-11T20:40:24,075 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:24,075 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/B is initiating minor compaction (all files) 2024-11-11T20:40:24,075 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/B in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:24,075 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4ecd39684fe84e7e8a599dec1cb0268d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a08d4f4e8988458095ad7ae18ae56626, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/5a8c705c9570425dab468c58f4178952, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9e80cb4f59364083bb11039d16779a82] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=48.5 K 2024-11-11T20:40:24,075 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 840e30c2da9a4b659afbd67c462ebe7a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357622104 2024-11-11T20:40:24,075 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ecd39684fe84e7e8a599dec1cb0268d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731357621899 2024-11-11T20:40:24,077 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22002402b612457787c51b3da75cbae7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731357622302 2024-11-11T20:40:24,077 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a08d4f4e8988458095ad7ae18ae56626, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357622104 2024-11-11T20:40:24,077 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab15c9b655504d19b36bdeeaa1c22406, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731357623482 2024-11-11T20:40:24,077 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a8c705c9570425dab468c58f4178952, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731357622302 2024-11-11T20:40:24,078 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e80cb4f59364083bb11039d16779a82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731357623482 2024-11-11T20:40:24,113 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:24,124 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:24,124 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/9bdcda93145e468ab04607f07fd5b7c9 is 50, key is test_row_0/B:col10/1731357623482/Put/seqid=0 2024-11-11T20:40:24,126 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411114e5349b96c044c3abd4562a179e9516a_3c30a36b246285db6085785ae9fb5d59 store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:24,129 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411114e5349b96c044c3abd4562a179e9516a_3c30a36b246285db6085785ae9fb5d59, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:24,129 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114e5349b96c044c3abd4562a179e9516a_3c30a36b246285db6085785ae9fb5d59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:24,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742035_1211 (size=4469) 2024-11-11T20:40:24,140 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#A#compaction#174 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:24,141 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/f81ed4b1ae644464a270c9c392d0335f is 175, key is test_row_0/A:col10/1731357623482/Put/seqid=0 2024-11-11T20:40:24,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742036_1212 (size=13017) 2024-11-11T20:40:24,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742037_1213 (size=31971) 2024-11-11T20:40:24,155 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/9bdcda93145e468ab04607f07fd5b7c9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9bdcda93145e468ab04607f07fd5b7c9 2024-11-11T20:40:24,158 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/f81ed4b1ae644464a270c9c392d0335f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f81ed4b1ae644464a270c9c392d0335f 2024-11-11T20:40:24,165 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/B of 3c30a36b246285db6085785ae9fb5d59 into 9bdcda93145e468ab04607f07fd5b7c9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:24,165 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:24,165 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/B, priority=12, startTime=1731357624070; duration=0sec 2024-11-11T20:40:24,165 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:24,165 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:24,165 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:24,177 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:24,177 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/C is initiating minor compaction (all files) 2024-11-11T20:40:24,177 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/C in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:24,178 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dfb53b677c3c43548d3767786d2ee5e0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7d29d93069234baabdc4c2f3e6367334, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/830adb496c1d4730afda09ae061dc0c4] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=36.6 K 2024-11-11T20:40:24,178 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb53b677c3c43548d3767786d2ee5e0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357622104 2024-11-11T20:40:24,178 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d29d93069234baabdc4c2f3e6367334, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731357622302 2024-11-11T20:40:24,182 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 830adb496c1d4730afda09ae061dc0c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731357623482 2024-11-11T20:40:24,183 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/A of 3c30a36b246285db6085785ae9fb5d59 into f81ed4b1ae644464a270c9c392d0335f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:24,183 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:24,183 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/A, priority=12, startTime=1731357624069; duration=0sec 2024-11-11T20:40:24,183 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:24,183 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:24,196 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#C#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:24,197 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/c85408f9960e4a41b3d5d48fadc4228a is 50, key is test_row_0/C:col10/1731357623482/Put/seqid=0 2024-11-11T20:40:24,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T20:40:24,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:24,203 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T20:40:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T20:40:24,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742038_1214 (size=13017) 2024-11-11T20:40:24,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ab8baf85f7eb4d00baa41af65faca5b3_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357623613/Put/seqid=0 2024-11-11T20:40:24,234 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/c85408f9960e4a41b3d5d48fadc4228a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c85408f9960e4a41b3d5d48fadc4228a 2024-11-11T20:40:24,241 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/C of 3c30a36b246285db6085785ae9fb5d59 into c85408f9960e4a41b3d5d48fadc4228a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:24,241 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:24,241 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/C, priority=13, startTime=1731357624072; duration=0sec 2024-11-11T20:40:24,241 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:24,241 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:24,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:24,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742039_1215 (size=12454) 2024-11-11T20:40:24,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357684306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:24,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357684411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:24,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:24,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:24,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:24,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357684621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:24,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:24,660 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ab8baf85f7eb4d00baa41af65faca5b3_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ab8baf85f7eb4d00baa41af65faca5b3_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/5d12aa51897d4d74b1adeab63254a065, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:24,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/5d12aa51897d4d74b1adeab63254a065 is 175, key is test_row_0/A:col10/1731357623613/Put/seqid=0 2024-11-11T20:40:24,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742040_1216 (size=31255) 2024-11-11T20:40:24,669 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/5d12aa51897d4d74b1adeab63254a065 2024-11-11T20:40:24,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/ad89596c0eca4773a85a4aabaf40a0f7 is 50, key is test_row_0/B:col10/1731357623613/Put/seqid=0 2024-11-11T20:40:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T20:40:24,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742041_1217 (size=12301) 2024-11-11T20:40:24,723 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/ad89596c0eca4773a85a4aabaf40a0f7 2024-11-11T20:40:24,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/738fca3607ac4bf3a25ba9423d6a9af6 is 50, key is test_row_0/C:col10/1731357623613/Put/seqid=0 2024-11-11T20:40:24,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742042_1218 (size=12301) 2024-11-11T20:40:24,765 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/738fca3607ac4bf3a25ba9423d6a9af6 2024-11-11T20:40:24,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/5d12aa51897d4d74b1adeab63254a065 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/5d12aa51897d4d74b1adeab63254a065 2024-11-11T20:40:24,779 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/5d12aa51897d4d74b1adeab63254a065, entries=150, sequenceid=329, filesize=30.5 K 2024-11-11T20:40:24,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/ad89596c0eca4773a85a4aabaf40a0f7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ad89596c0eca4773a85a4aabaf40a0f7 2024-11-11T20:40:24,786 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ad89596c0eca4773a85a4aabaf40a0f7, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T20:40:24,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/738fca3607ac4bf3a25ba9423d6a9af6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/738fca3607ac4bf3a25ba9423d6a9af6 2024-11-11T20:40:24,803 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/738fca3607ac4bf3a25ba9423d6a9af6, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T20:40:24,805 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3c30a36b246285db6085785ae9fb5d59 in 602ms, sequenceid=329, compaction requested=false 2024-11-11T20:40:24,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:24,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:24,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-11T20:40:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-11T20:40:24,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T20:40:24,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2220 sec 2024-11-11T20:40:24,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.2300 sec 2024-11-11T20:40:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:24,935 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:24,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T20:40:24,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f81313fb32864494954eb0a43e5fde39_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357624293/Put/seqid=0 2024-11-11T20:40:24,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742043_1219 (size=12454) 2024-11-11T20:40:24,980 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:24,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f81313fb32864494954eb0a43e5fde39_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f81313fb32864494954eb0a43e5fde39_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:24,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/45f8f9ad592643f190254585afd66cd7, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:24,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/45f8f9ad592643f190254585afd66cd7 is 175, key is test_row_0/A:col10/1731357624293/Put/seqid=0 2024-11-11T20:40:24,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742044_1220 (size=31255) 2024-11-11T20:40:24,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/45f8f9ad592643f190254585afd66cd7 2024-11-11T20:40:25,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/a4b7c91654434bc4a95071756edaa134 is 50, key is test_row_0/B:col10/1731357624293/Put/seqid=0 2024-11-11T20:40:25,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742045_1221 (size=12301) 2024-11-11T20:40:25,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/a4b7c91654434bc4a95071756edaa134 2024-11-11T20:40:25,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357685019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:25,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/8b8b1c11ded44accb418d56b1dd01300 is 50, key is test_row_0/C:col10/1731357624293/Put/seqid=0 2024-11-11T20:40:25,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742046_1222 (size=12301) 2024-11-11T20:40:25,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/8b8b1c11ded44accb418d56b1dd01300 2024-11-11T20:40:25,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/45f8f9ad592643f190254585afd66cd7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/45f8f9ad592643f190254585afd66cd7 2024-11-11T20:40:25,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/45f8f9ad592643f190254585afd66cd7, entries=150, sequenceid=347, filesize=30.5 K 2024-11-11T20:40:25,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/a4b7c91654434bc4a95071756edaa134 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a4b7c91654434bc4a95071756edaa134 2024-11-11T20:40:25,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a4b7c91654434bc4a95071756edaa134, entries=150, sequenceid=347, filesize=12.0 K 2024-11-11T20:40:25,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/8b8b1c11ded44accb418d56b1dd01300 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/8b8b1c11ded44accb418d56b1dd01300 2024-11-11T20:40:25,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/8b8b1c11ded44accb418d56b1dd01300, entries=150, sequenceid=347, filesize=12.0 K 2024-11-11T20:40:25,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 3c30a36b246285db6085785ae9fb5d59 in 116ms, sequenceid=347, compaction requested=true 2024-11-11T20:40:25,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:25,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:25,051 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:25,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:25,051 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:25,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:25,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:25,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:25,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:25,052 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:25,053 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/A is initiating minor compaction (all files) 2024-11-11T20:40:25,053 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/A in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:25,053 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f81ed4b1ae644464a270c9c392d0335f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/5d12aa51897d4d74b1adeab63254a065, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/45f8f9ad592643f190254585afd66cd7] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=92.3 K 2024-11-11T20:40:25,053 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:25,053 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f81ed4b1ae644464a270c9c392d0335f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/5d12aa51897d4d74b1adeab63254a065, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/45f8f9ad592643f190254585afd66cd7] 2024-11-11T20:40:25,053 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:25,053 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/B is initiating minor compaction (all files) 2024-11-11T20:40:25,053 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/B in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:25,053 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9bdcda93145e468ab04607f07fd5b7c9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ad89596c0eca4773a85a4aabaf40a0f7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a4b7c91654434bc4a95071756edaa134] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=36.7 K 2024-11-11T20:40:25,054 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bdcda93145e468ab04607f07fd5b7c9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731357623482 2024-11-11T20:40:25,054 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f81ed4b1ae644464a270c9c392d0335f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731357623482 2024-11-11T20:40:25,054 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ad89596c0eca4773a85a4aabaf40a0f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357623588 2024-11-11T20:40:25,054 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d12aa51897d4d74b1adeab63254a065, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357623588 2024-11-11T20:40:25,054 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a4b7c91654434bc4a95071756edaa134, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731357624293 2024-11-11T20:40:25,055 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45f8f9ad592643f190254585afd66cd7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731357624293 2024-11-11T20:40:25,073 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#B#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:25,073 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/3663d50c551249d383130e1b64acfda3 is 50, key is test_row_0/B:col10/1731357624293/Put/seqid=0 2024-11-11T20:40:25,074 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:25,082 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111037f19c88df944cf90b216a2667f824b_3c30a36b246285db6085785ae9fb5d59 store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:25,085 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111037f19c88df944cf90b216a2667f824b_3c30a36b246285db6085785ae9fb5d59, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:25,085 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111037f19c88df944cf90b216a2667f824b_3c30a36b246285db6085785ae9fb5d59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:25,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742047_1223 (size=13119) 2024-11-11T20:40:25,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742048_1224 (size=4469) 2024-11-11T20:40:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:25,135 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:25,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T20:40:25,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e1199cb9061f4b84bfdbbb198d86b271_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357625133/Put/seqid=0 2024-11-11T20:40:25,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742049_1225 (size=14994) 2024-11-11T20:40:25,168 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:25,172 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e1199cb9061f4b84bfdbbb198d86b271_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e1199cb9061f4b84bfdbbb198d86b271_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:25,173 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/15d314e7b1514157a3a387c6695f2ce3, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:25,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/15d314e7b1514157a3a387c6695f2ce3 is 175, key is test_row_0/A:col10/1731357625133/Put/seqid=0 2024-11-11T20:40:25,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742050_1226 (size=39949) 2024-11-11T20:40:25,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/15d314e7b1514157a3a387c6695f2ce3 2024-11-11T20:40:25,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/fedc5d48b298472a8b6fac0bd7c22b6e is 50, key is test_row_0/B:col10/1731357625133/Put/seqid=0 2024-11-11T20:40:25,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357685211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:25,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742051_1227 (size=12301) 2024-11-11T20:40:25,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357685321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:25,513 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:25,514 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/f86a28f9d2c94fbdb6350e275ea552cf is 175, key is test_row_0/A:col10/1731357624293/Put/seqid=0 2024-11-11T20:40:25,516 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/3663d50c551249d383130e1b64acfda3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3663d50c551249d383130e1b64acfda3 2024-11-11T20:40:25,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742052_1228 (size=32073) 2024-11-11T20:40:25,521 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/B of 3c30a36b246285db6085785ae9fb5d59 into 3663d50c551249d383130e1b64acfda3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:25,521 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:25,521 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/B, priority=13, startTime=1731357625051; duration=0sec 2024-11-11T20:40:25,521 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:25,521 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:25,521 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:25,523 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:25,523 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/C is initiating minor compaction (all files) 2024-11-11T20:40:25,523 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/C in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:25,523 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c85408f9960e4a41b3d5d48fadc4228a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/738fca3607ac4bf3a25ba9423d6a9af6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/8b8b1c11ded44accb418d56b1dd01300] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=36.7 K 2024-11-11T20:40:25,524 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c85408f9960e4a41b3d5d48fadc4228a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731357623482 2024-11-11T20:40:25,524 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 738fca3607ac4bf3a25ba9423d6a9af6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357623588 2024-11-11T20:40:25,525 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b8b1c11ded44accb418d56b1dd01300, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731357624293 2024-11-11T20:40:25,533 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#C#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:25,533 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/68cbaa20742a43e8a062f0eb7c866cc1 is 50, key is test_row_0/C:col10/1731357624293/Put/seqid=0 2024-11-11T20:40:25,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357685531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:25,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742053_1229 (size=13119) 2024-11-11T20:40:25,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/fedc5d48b298472a8b6fac0bd7c22b6e 2024-11-11T20:40:25,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/0bcd7ef8609b4952ab47b9107676c34c is 50, key is test_row_0/C:col10/1731357625133/Put/seqid=0 2024-11-11T20:40:25,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742054_1230 (size=12301) 2024-11-11T20:40:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T20:40:25,731 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:25,732 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-11T20:40:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:40:25,734 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:25,734 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:25,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:40:25,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357685841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:25,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:40:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:25,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:25,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:25,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:25,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:25,921 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/f86a28f9d2c94fbdb6350e275ea552cf as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f86a28f9d2c94fbdb6350e275ea552cf 2024-11-11T20:40:25,926 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/A of 3c30a36b246285db6085785ae9fb5d59 into f86a28f9d2c94fbdb6350e275ea552cf(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:25,926 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:25,926 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/A, priority=13, startTime=1731357625051; duration=0sec 2024-11-11T20:40:25,926 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:25,926 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:25,941 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/68cbaa20742a43e8a062f0eb7c866cc1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/68cbaa20742a43e8a062f0eb7c866cc1 2024-11-11T20:40:25,950 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/C of 3c30a36b246285db6085785ae9fb5d59 into 68cbaa20742a43e8a062f0eb7c866cc1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:25,950 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:25,950 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/C, priority=13, startTime=1731357625051; duration=0sec 2024-11-11T20:40:25,950 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:25,950 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:26,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:40:26,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:26,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:26,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:26,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:26,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:26,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/0bcd7ef8609b4952ab47b9107676c34c 2024-11-11T20:40:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:40:26,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/15d314e7b1514157a3a387c6695f2ce3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/15d314e7b1514157a3a387c6695f2ce3 2024-11-11T20:40:26,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/15d314e7b1514157a3a387c6695f2ce3, entries=200, sequenceid=367, filesize=39.0 K 2024-11-11T20:40:26,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/fedc5d48b298472a8b6fac0bd7c22b6e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fedc5d48b298472a8b6fac0bd7c22b6e 2024-11-11T20:40:26,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fedc5d48b298472a8b6fac0bd7c22b6e, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T20:40:26,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/0bcd7ef8609b4952ab47b9107676c34c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/0bcd7ef8609b4952ab47b9107676c34c 2024-11-11T20:40:26,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/0bcd7ef8609b4952ab47b9107676c34c, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T20:40:26,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3c30a36b246285db6085785ae9fb5d59 in 953ms, sequenceid=367, compaction requested=false 2024-11-11T20:40:26,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:26,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:40:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:26,193 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T20:40:26,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115d6db297ef1a4985840fa4b7ceb43dcc_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357625209/Put/seqid=0 2024-11-11T20:40:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742055_1231 (size=12454) 2024-11-11T20:40:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:40:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:26,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:26,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357686456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:26,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357686571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:26,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:26,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:26,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:26,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:26,616 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115d6db297ef1a4985840fa4b7ceb43dcc_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d6db297ef1a4985840fa4b7ceb43dcc_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:26,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/a07d9cd017db4a7fb5899acd825956d9, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:26,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/a07d9cd017db4a7fb5899acd825956d9 is 175, key is test_row_0/A:col10/1731357625209/Put/seqid=0 2024-11-11T20:40:26,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742056_1232 (size=31255) 2024-11-11T20:40:26,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357686792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:40:27,034 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/a07d9cd017db4a7fb5899acd825956d9 2024-11-11T20:40:27,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/212b5aecbfa247da8aa20899205f6820 is 50, key is test_row_0/B:col10/1731357625209/Put/seqid=0 2024-11-11T20:40:27,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742057_1233 (size=12301) 2024-11-11T20:40:27,049 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/212b5aecbfa247da8aa20899205f6820 2024-11-11T20:40:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/45d1295751584418921b036975351bdc is 50, key is test_row_0/C:col10/1731357625209/Put/seqid=0 2024-11-11T20:40:27,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742058_1234 (size=12301) 2024-11-11T20:40:27,075 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/45d1295751584418921b036975351bdc 2024-11-11T20:40:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/a07d9cd017db4a7fb5899acd825956d9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a07d9cd017db4a7fb5899acd825956d9 2024-11-11T20:40:27,086 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a07d9cd017db4a7fb5899acd825956d9, entries=150, sequenceid=386, filesize=30.5 K 2024-11-11T20:40:27,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/212b5aecbfa247da8aa20899205f6820 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/212b5aecbfa247da8aa20899205f6820 2024-11-11T20:40:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54030 deadline: 1731357687080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,088 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_0' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53954 deadline: 1731357687091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,093 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in 3c30a36b246285db6085785ae9fb5d59 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:40:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,097 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/212b5aecbfa247da8aa20899205f6820, entries=150, sequenceid=386, filesize=12.0 K 2024-11-11T20:40:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/45d1295751584418921b036975351bdc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/45d1295751584418921b036975351bdc 2024-11-11T20:40:27,102 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/45d1295751584418921b036975351bdc, entries=150, sequenceid=386, filesize=12.0 K 2024-11-11T20:40:27,103 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3c30a36b246285db6085785ae9fb5d59 in 910ms, sequenceid=386, compaction requested=true 2024-11-11T20:40:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-11T20:40:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-11T20:40:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:27,105 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:27,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:40:27,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T20:40:27,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3700 sec 2024-11-11T20:40:27,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.3740 sec 2024-11-11T20:40:27,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d60872dc51a74e3498272d20f77211b0_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357627104/Put/seqid=0 2024-11-11T20:40:27,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742059_1235 (size=17534) 2024-11-11T20:40:27,127 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357687125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357687128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357687130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d60872dc51a74e3498272d20f77211b0_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d60872dc51a74e3498272d20f77211b0_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:27,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/358d16cca5f34915a1bba1e952a05db3, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:27,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/358d16cca5f34915a1bba1e952a05db3 is 175, key is test_row_0/A:col10/1731357627104/Put/seqid=0 2024-11-11T20:40:27,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742060_1236 (size=48639) 2024-11-11T20:40:27,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357687241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357687241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357687241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357687450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357687451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357687451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/358d16cca5f34915a1bba1e952a05db3 2024-11-11T20:40:27,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/522cd9a00a4a43dabca8af2449678382 is 50, key is test_row_0/B:col10/1731357627104/Put/seqid=0 2024-11-11T20:40:27,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742061_1237 (size=12301) 2024-11-11T20:40:27,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357687761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357687771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357687771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:40:27,890 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:27,891 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-11T20:40:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:40:27,893 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:27,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:27,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:27,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/522cd9a00a4a43dabca8af2449678382 2024-11-11T20:40:27,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/cb74b9ff09524423b8905c944fe7520e is 50, key is test_row_0/C:col10/1731357627104/Put/seqid=0 2024-11-11T20:40:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:40:28,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742062_1238 (size=12301) 2024-11-11T20:40:28,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/cb74b9ff09524423b8905c944fe7520e 2024-11-11T20:40:28,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/358d16cca5f34915a1bba1e952a05db3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/358d16cca5f34915a1bba1e952a05db3 2024-11-11T20:40:28,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/358d16cca5f34915a1bba1e952a05db3, entries=250, sequenceid=408, filesize=47.5 K 2024-11-11T20:40:28,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/522cd9a00a4a43dabca8af2449678382 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/522cd9a00a4a43dabca8af2449678382 2024-11-11T20:40:28,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/522cd9a00a4a43dabca8af2449678382, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T20:40:28,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/cb74b9ff09524423b8905c944fe7520e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/cb74b9ff09524423b8905c944fe7520e 2024-11-11T20:40:28,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/cb74b9ff09524423b8905c944fe7520e, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T20:40:28,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3c30a36b246285db6085785ae9fb5d59 in 919ms, sequenceid=408, compaction requested=true 2024-11-11T20:40:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:28,025 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:28,025 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:28,026 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:28,026 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:28,026 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/A is initiating minor compaction (all files) 2024-11-11T20:40:28,026 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/B is initiating minor compaction (all files) 2024-11-11T20:40:28,026 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/B in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:28,026 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/A in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:28,027 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f86a28f9d2c94fbdb6350e275ea552cf, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/15d314e7b1514157a3a387c6695f2ce3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a07d9cd017db4a7fb5899acd825956d9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/358d16cca5f34915a1bba1e952a05db3] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=148.4 K 2024-11-11T20:40:28,027 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3663d50c551249d383130e1b64acfda3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fedc5d48b298472a8b6fac0bd7c22b6e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/212b5aecbfa247da8aa20899205f6820, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/522cd9a00a4a43dabca8af2449678382] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=48.8 K 2024-11-11T20:40:28,027 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:28,027 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f86a28f9d2c94fbdb6350e275ea552cf, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/15d314e7b1514157a3a387c6695f2ce3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a07d9cd017db4a7fb5899acd825956d9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/358d16cca5f34915a1bba1e952a05db3] 2024-11-11T20:40:28,027 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3663d50c551249d383130e1b64acfda3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731357624293 2024-11-11T20:40:28,027 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fedc5d48b298472a8b6fac0bd7c22b6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731357625007 2024-11-11T20:40:28,027 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 212b5aecbfa247da8aa20899205f6820, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731357625171 2024-11-11T20:40:28,028 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 522cd9a00a4a43dabca8af2449678382, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731357626454 2024-11-11T20:40:28,029 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86a28f9d2c94fbdb6350e275ea552cf, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731357624293 2024-11-11T20:40:28,029 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d314e7b1514157a3a387c6695f2ce3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731357625007 2024-11-11T20:40:28,029 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a07d9cd017db4a7fb5899acd825956d9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731357625171 2024-11-11T20:40:28,030 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 358d16cca5f34915a1bba1e952a05db3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731357626429 2024-11-11T20:40:28,035 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:28,036 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:28,037 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/331b7ceeb6f74ff7898a36ed780d67d8 is 50, key is test_row_0/B:col10/1731357627104/Put/seqid=0 2024-11-11T20:40:28,037 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111d93927e351fd47bab7e6790dcb38bd03_3c30a36b246285db6085785ae9fb5d59 store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:28,039 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111d93927e351fd47bab7e6790dcb38bd03_3c30a36b246285db6085785ae9fb5d59, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:28,039 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d93927e351fd47bab7e6790dcb38bd03_3c30a36b246285db6085785ae9fb5d59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:28,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742063_1239 (size=13255) 2024-11-11T20:40:28,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742064_1240 (size=4469) 2024-11-11T20:40:28,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:40:28,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:28,047 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T20:40:28,049 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/331b7ceeb6f74ff7898a36ed780d67d8 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/331b7ceeb6f74ff7898a36ed780d67d8 2024-11-11T20:40:28,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e60290d1b5fc4af6988ec60b857c6e9b_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357627128/Put/seqid=0 2024-11-11T20:40:28,054 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/B of 3c30a36b246285db6085785ae9fb5d59 into 331b7ceeb6f74ff7898a36ed780d67d8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:28,054 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:28,055 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/B, priority=12, startTime=1731357628025; duration=0sec 2024-11-11T20:40:28,055 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:28,055 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:28,055 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:28,056 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:28,056 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/C is initiating minor compaction (all files) 2024-11-11T20:40:28,056 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/C in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:28,057 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/68cbaa20742a43e8a062f0eb7c866cc1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/0bcd7ef8609b4952ab47b9107676c34c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/45d1295751584418921b036975351bdc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/cb74b9ff09524423b8905c944fe7520e] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=48.8 K 2024-11-11T20:40:28,058 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 68cbaa20742a43e8a062f0eb7c866cc1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731357624293 2024-11-11T20:40:28,059 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bcd7ef8609b4952ab47b9107676c34c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731357625007 2024-11-11T20:40:28,059 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d1295751584418921b036975351bdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731357625171 2024-11-11T20:40:28,059 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting cb74b9ff09524423b8905c944fe7520e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731357626454 2024-11-11T20:40:28,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742065_1241 (size=12454) 2024-11-11T20:40:28,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:28,083 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e60290d1b5fc4af6988ec60b857c6e9b_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e60290d1b5fc4af6988ec60b857c6e9b_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:28,083 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:28,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/93602c877ca943d28f646fd45318f728, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:28,084 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/6a73e418e980451c9c95afd56294a20d is 50, key is test_row_0/C:col10/1731357627104/Put/seqid=0 2024-11-11T20:40:28,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/93602c877ca943d28f646fd45318f728 is 175, key is test_row_0/A:col10/1731357627128/Put/seqid=0 2024-11-11T20:40:28,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742067_1243 (size=31255) 2024-11-11T20:40:28,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742066_1242 (size=13255) 2024-11-11T20:40:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:40:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:28,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:28,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357688315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357688316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357688318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357688421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357688421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357688430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,444 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#A#compaction#196 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:28,445 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/7908ed6ae49c4586809eb5e8dfe7cd71 is 175, key is test_row_0/A:col10/1731357627104/Put/seqid=0 2024-11-11T20:40:28,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742068_1244 (size=32209) 2024-11-11T20:40:28,478 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/7908ed6ae49c4586809eb5e8dfe7cd71 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/7908ed6ae49c4586809eb5e8dfe7cd71 2024-11-11T20:40:28,486 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/A of 3c30a36b246285db6085785ae9fb5d59 into 7908ed6ae49c4586809eb5e8dfe7cd71(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:28,486 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:28,486 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/A, priority=12, startTime=1731357628024; duration=0sec 2024-11-11T20:40:28,486 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:28,486 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:28,497 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/93602c877ca943d28f646fd45318f728 2024-11-11T20:40:28,504 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/6a73e418e980451c9c95afd56294a20d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6a73e418e980451c9c95afd56294a20d 2024-11-11T20:40:28,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/0a6fc7ae1f404410ac023b79ca2ba172 is 50, key is test_row_0/B:col10/1731357627128/Put/seqid=0 2024-11-11T20:40:28,509 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/C of 3c30a36b246285db6085785ae9fb5d59 into 6a73e418e980451c9c95afd56294a20d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:28,509 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:28,509 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/C, priority=12, startTime=1731357628025; duration=0sec 2024-11-11T20:40:28,509 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:28,509 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:28,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742069_1245 (size=12301) 2024-11-11T20:40:28,519 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/0a6fc7ae1f404410ac023b79ca2ba172 2024-11-11T20:40:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:40:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e8fa73eba0c844dab34ec3295d5b993d is 50, key is test_row_0/C:col10/1731357627128/Put/seqid=0 2024-11-11T20:40:28,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742070_1246 (size=12301) 2024-11-11T20:40:28,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357688630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357688630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357688641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,942 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e8fa73eba0c844dab34ec3295d5b993d 2024-11-11T20:40:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357688941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357688941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/93602c877ca943d28f646fd45318f728 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/93602c877ca943d28f646fd45318f728 2024-11-11T20:40:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357688950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:28,952 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/93602c877ca943d28f646fd45318f728, entries=150, sequenceid=422, filesize=30.5 K 2024-11-11T20:40:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:28,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/0a6fc7ae1f404410ac023b79ca2ba172 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/0a6fc7ae1f404410ac023b79ca2ba172 2024-11-11T20:40:28,959 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/0a6fc7ae1f404410ac023b79ca2ba172, entries=150, sequenceid=422, filesize=12.0 K 2024-11-11T20:40:28,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/e8fa73eba0c844dab34ec3295d5b993d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e8fa73eba0c844dab34ec3295d5b993d 2024-11-11T20:40:28,964 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e8fa73eba0c844dab34ec3295d5b993d, entries=150, sequenceid=422, filesize=12.0 K 2024-11-11T20:40:28,965 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3c30a36b246285db6085785ae9fb5d59 in 918ms, sequenceid=422, compaction requested=false 2024-11-11T20:40:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-11T20:40:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-11T20:40:28,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T20:40:28,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0730 sec 2024-11-11T20:40:28,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.0770 sec 2024-11-11T20:40:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:40:29,031 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:29,031 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-11T20:40:29,033 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:29,034 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:40:29,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:40:29,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T20:40:29,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:29,188 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:40:29,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111560a246e43394173bd26fdfa308a5ef3_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357628305/Put/seqid=0 2024-11-11T20:40:29,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742071_1247 (size=12454) 2024-11-11T20:40:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:40:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:29,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:29,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357689464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:29,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357689466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357689467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357689571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357689571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357689571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:29,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:29,627 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111560a246e43394173bd26fdfa308a5ef3_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111560a246e43394173bd26fdfa308a5ef3_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/fafab992e51146d5bcd68b65df067098, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/fafab992e51146d5bcd68b65df067098 is 175, key is test_row_0/A:col10/1731357628305/Put/seqid=0 2024-11-11T20:40:29,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742072_1248 (size=31255) 2024-11-11T20:40:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:40:29,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357689781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:29,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357689781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357689781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:30,043 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/fafab992e51146d5bcd68b65df067098 2024-11-11T20:40:30,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/49a18d1be9194ff08c6f6de475a2340b is 50, key is test_row_0/B:col10/1731357628305/Put/seqid=0 2024-11-11T20:40:30,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742073_1249 (size=12301) 2024-11-11T20:40:30,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357690090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:30,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357690090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:30,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357690090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:40:30,455 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/49a18d1be9194ff08c6f6de475a2340b 2024-11-11T20:40:30,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/be08332830494a5b98105ee8fe7fc690 is 50, key is test_row_0/C:col10/1731357628305/Put/seqid=0 2024-11-11T20:40:30,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742074_1250 (size=12301) 2024-11-11T20:40:30,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:30,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357690601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357690600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357690601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:30,868 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/be08332830494a5b98105ee8fe7fc690 2024-11-11T20:40:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/fafab992e51146d5bcd68b65df067098 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/fafab992e51146d5bcd68b65df067098 2024-11-11T20:40:30,877 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/fafab992e51146d5bcd68b65df067098, entries=150, sequenceid=447, filesize=30.5 K 2024-11-11T20:40:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/49a18d1be9194ff08c6f6de475a2340b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/49a18d1be9194ff08c6f6de475a2340b 2024-11-11T20:40:30,882 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/49a18d1be9194ff08c6f6de475a2340b, entries=150, sequenceid=447, filesize=12.0 K 2024-11-11T20:40:30,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/be08332830494a5b98105ee8fe7fc690 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/be08332830494a5b98105ee8fe7fc690 2024-11-11T20:40:30,887 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/be08332830494a5b98105ee8fe7fc690, entries=150, sequenceid=447, filesize=12.0 K 2024-11-11T20:40:30,887 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3c30a36b246285db6085785ae9fb5d59 in 1700ms, sequenceid=447, compaction requested=true 2024-11-11T20:40:30,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:30,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:30,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-11T20:40:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-11T20:40:30,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T20:40:30,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8550 sec 2024-11-11T20:40:30,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.8600 sec 2024-11-11T20:40:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:40:31,182 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:31,182 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-11T20:40:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:40:31,185 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:31,185 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:31,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:40:31,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T20:40:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:31,338 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T20:40:31,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114e04f7112d9e41d898fd1a7ee3933a1c_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357629461/Put/seqid=0 2024-11-11T20:40:31,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742075_1251 (size=12454) 2024-11-11T20:40:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:40:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:31,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. as already flushing 2024-11-11T20:40:31,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357691649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:31,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357691652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:31,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357691653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:31,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:31,759 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114e04f7112d9e41d898fd1a7ee3933a1c_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114e04f7112d9e41d898fd1a7ee3933a1c_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:31,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/9cc493101ac94a9ebdbf4ab2f81a3e37, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:31,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/9cc493101ac94a9ebdbf4ab2f81a3e37 is 175, key is test_row_0/A:col10/1731357629461/Put/seqid=0 2024-11-11T20:40:31,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357691761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:31,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357691761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:31,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742076_1252 (size=31255) 2024-11-11T20:40:31,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357691771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:40:31,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:31,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357691971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:31,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357691971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:31,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357691981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:32,172 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=461, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/9cc493101ac94a9ebdbf4ab2f81a3e37 2024-11-11T20:40:32,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/4515adfae0274642bb82e503ec4cb057 is 50, key is test_row_0/B:col10/1731357629461/Put/seqid=0 2024-11-11T20:40:32,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742077_1253 (size=12301) 2024-11-11T20:40:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357692281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:32,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357692281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357692290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:40:32,604 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/4515adfae0274642bb82e503ec4cb057 2024-11-11T20:40:32,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/7513527a395d4533be70534c4ff72361 is 50, key is test_row_0/C:col10/1731357629461/Put/seqid=0 2024-11-11T20:40:32,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742078_1254 (size=12301) 2024-11-11T20:40:32,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53966 deadline: 1731357692790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:32,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1731357692790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:32,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1731357692801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:40:33,016 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/7513527a395d4533be70534c4ff72361 2024-11-11T20:40:33,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/9cc493101ac94a9ebdbf4ab2f81a3e37 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/9cc493101ac94a9ebdbf4ab2f81a3e37 2024-11-11T20:40:33,023 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/9cc493101ac94a9ebdbf4ab2f81a3e37, entries=150, sequenceid=461, filesize=30.5 K 2024-11-11T20:40:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/4515adfae0274642bb82e503ec4cb057 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4515adfae0274642bb82e503ec4cb057 2024-11-11T20:40:33,028 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4515adfae0274642bb82e503ec4cb057, entries=150, sequenceid=461, filesize=12.0 K 2024-11-11T20:40:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/7513527a395d4533be70534c4ff72361 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7513527a395d4533be70534c4ff72361 2024-11-11T20:40:33,032 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7513527a395d4533be70534c4ff72361, entries=150, sequenceid=461, filesize=12.0 K 2024-11-11T20:40:33,033 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3c30a36b246285db6085785ae9fb5d59 in 1695ms, sequenceid=461, compaction requested=true 2024-11-11T20:40:33,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:33,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:33,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-11T20:40:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-11T20:40:33,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T20:40:33,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8480 sec 2024-11-11T20:40:33,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.8530 sec 2024-11-11T20:40:33,225 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-11T20:40:33,225 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:33,225 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,225 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:33,226 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-11T20:40:33,226 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:33,226 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,226 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:33,228 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-11T20:40:33,228 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:33,228 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,228 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:33,232 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-11T20:40:33,232 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:33,232 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,232 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:40:33,332 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:33,802 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:33,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T20:40:33,802 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-11T20:40:33,802 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:33,802 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,802 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:33,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aca1ec88206b40b0a1c535f85894bc70_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357631645/Put/seqid=0 2024-11-11T20:40:33,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742079_1255 (size=12454) 2024-11-11T20:40:33,812 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-11-11T20:40:33,812 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-11-11T20:40:33,812 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:33,812 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:33,812 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,812 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,812 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,812 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:33,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:33,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:34,213 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:34,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aca1ec88206b40b0a1c535f85894bc70_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aca1ec88206b40b0a1c535f85894bc70_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:34,225 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b348206556dd46e888bc838760bb346a, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:34,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b348206556dd46e888bc838760bb346a is 175, key is test_row_0/A:col10/1731357631645/Put/seqid=0 2024-11-11T20:40:34,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742080_1256 (size=31255) 2024-11-11T20:40:34,631 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=484, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b348206556dd46e888bc838760bb346a 2024-11-11T20:40:34,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/f46dad21d99d453a9aeb3328d711661e is 50, key is test_row_0/B:col10/1731357631645/Put/seqid=0 2024-11-11T20:40:34,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742081_1257 (size=12301) 2024-11-11T20:40:35,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/f46dad21d99d453a9aeb3328d711661e 2024-11-11T20:40:35,052 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:40:35,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/6407f8b1e4074ef0b199df5fd243cd93 is 50, key is test_row_0/C:col10/1731357631645/Put/seqid=0 2024-11-11T20:40:35,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742082_1258 (size=12301) 2024-11-11T20:40:35,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/6407f8b1e4074ef0b199df5fd243cd93 2024-11-11T20:40:35,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b348206556dd46e888bc838760bb346a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b348206556dd46e888bc838760bb346a 2024-11-11T20:40:35,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b348206556dd46e888bc838760bb346a, entries=150, sequenceid=484, filesize=30.5 K 2024-11-11T20:40:35,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/f46dad21d99d453a9aeb3328d711661e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f46dad21d99d453a9aeb3328d711661e 2024-11-11T20:40:35,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f46dad21d99d453a9aeb3328d711661e, entries=150, sequenceid=484, filesize=12.0 K 2024-11-11T20:40:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/6407f8b1e4074ef0b199df5fd243cd93 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6407f8b1e4074ef0b199df5fd243cd93 2024-11-11T20:40:35,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6407f8b1e4074ef0b199df5fd243cd93, entries=150, sequenceid=484, filesize=12.0 K 2024-11-11T20:40:35,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=13.42 KB/13740 for 3c30a36b246285db6085785ae9fb5d59 in 1680ms, sequenceid=484, compaction requested=true 2024-11-11T20:40:35,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:35,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:35,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:35,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:35,483 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T20:40:35,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:35,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c30a36b246285db6085785ae9fb5d59:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:35,483 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T20:40:35,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:35,484 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62459 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T20:40:35,484 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 157229 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T20:40:35,484 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/A is initiating minor compaction (all files) 2024-11-11T20:40:35,484 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/B is initiating minor compaction (all files) 2024-11-11T20:40:35,485 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/A in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:35,485 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/B in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:35,485 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/7908ed6ae49c4586809eb5e8dfe7cd71, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/93602c877ca943d28f646fd45318f728, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/fafab992e51146d5bcd68b65df067098, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/9cc493101ac94a9ebdbf4ab2f81a3e37, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b348206556dd46e888bc838760bb346a] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=153.5 K 2024-11-11T20:40:35,485 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/331b7ceeb6f74ff7898a36ed780d67d8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/0a6fc7ae1f404410ac023b79ca2ba172, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/49a18d1be9194ff08c6f6de475a2340b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4515adfae0274642bb82e503ec4cb057, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f46dad21d99d453a9aeb3328d711661e] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=61.0 K 2024-11-11T20:40:35,485 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:35,485 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/7908ed6ae49c4586809eb5e8dfe7cd71, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/93602c877ca943d28f646fd45318f728, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/fafab992e51146d5bcd68b65df067098, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/9cc493101ac94a9ebdbf4ab2f81a3e37, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b348206556dd46e888bc838760bb346a] 2024-11-11T20:40:35,485 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 331b7ceeb6f74ff7898a36ed780d67d8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731357626454 2024-11-11T20:40:35,485 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7908ed6ae49c4586809eb5e8dfe7cd71, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731357626454 2024-11-11T20:40:35,486 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93602c877ca943d28f646fd45318f728, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731357627116 2024-11-11T20:40:35,486 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6fc7ae1f404410ac023b79ca2ba172, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731357627116 2024-11-11T20:40:35,486 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fafab992e51146d5bcd68b65df067098, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731357628305 2024-11-11T20:40:35,486 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a18d1be9194ff08c6f6de475a2340b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731357628305 2024-11-11T20:40:35,486 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc493101ac94a9ebdbf4ab2f81a3e37, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731357629461 2024-11-11T20:40:35,486 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4515adfae0274642bb82e503ec4cb057, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731357629461 2024-11-11T20:40:35,487 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b348206556dd46e888bc838760bb346a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731357631645 2024-11-11T20:40:35,487 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f46dad21d99d453a9aeb3328d711661e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731357631645 2024-11-11T20:40:35,500 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:35,502 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111151d3ab6ab0fb4e42b925147017322d64_3c30a36b246285db6085785ae9fb5d59 store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:35,502 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#B#compaction#210 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:35,503 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/5a8d8353605a4ed1840eeba8024b172e is 50, key is test_row_0/B:col10/1731357631645/Put/seqid=0 2024-11-11T20:40:35,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742083_1259 (size=13425) 2024-11-11T20:40:35,538 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111151d3ab6ab0fb4e42b925147017322d64_3c30a36b246285db6085785ae9fb5d59, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:35,538 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111151d3ab6ab0fb4e42b925147017322d64_3c30a36b246285db6085785ae9fb5d59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:35,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742084_1260 (size=4469) 2024-11-11T20:40:35,920 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/5a8d8353605a4ed1840eeba8024b172e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/5a8d8353605a4ed1840eeba8024b172e 2024-11-11T20:40:35,927 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/B of 3c30a36b246285db6085785ae9fb5d59 into 5a8d8353605a4ed1840eeba8024b172e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:35,927 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:35,927 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/B, priority=11, startTime=1731357635483; duration=0sec 2024-11-11T20:40:35,927 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:35,927 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:B 2024-11-11T20:40:35,927 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T20:40:35,929 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62459 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T20:40:35,929 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 3c30a36b246285db6085785ae9fb5d59/C is initiating minor compaction (all files) 2024-11-11T20:40:35,929 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c30a36b246285db6085785ae9fb5d59/C in TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:35,929 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6a73e418e980451c9c95afd56294a20d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e8fa73eba0c844dab34ec3295d5b993d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/be08332830494a5b98105ee8fe7fc690, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7513527a395d4533be70534c4ff72361, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6407f8b1e4074ef0b199df5fd243cd93] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp, totalSize=61.0 K 2024-11-11T20:40:35,930 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a73e418e980451c9c95afd56294a20d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731357626454 2024-11-11T20:40:35,930 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting e8fa73eba0c844dab34ec3295d5b993d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731357627116 2024-11-11T20:40:35,931 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting be08332830494a5b98105ee8fe7fc690, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731357628305 2024-11-11T20:40:35,931 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7513527a395d4533be70534c4ff72361, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731357629461 2024-11-11T20:40:35,932 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6407f8b1e4074ef0b199df5fd243cd93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731357631645 2024-11-11T20:40:35,942 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#A#compaction#211 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:35,943 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c30a36b246285db6085785ae9fb5d59#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:35,943 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/dd206e7c2a2f4aa4a1aba4307ed58515 is 50, key is test_row_0/C:col10/1731357631645/Put/seqid=0 2024-11-11T20:40:35,943 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/e8559155c9454635b6ff4015bc7f7d65 is 175, key is test_row_0/A:col10/1731357631645/Put/seqid=0 2024-11-11T20:40:35,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742085_1261 (size=13425) 2024-11-11T20:40:35,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742086_1262 (size=32379) 2024-11-11T20:40:36,353 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/dd206e7c2a2f4aa4a1aba4307ed58515 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dd206e7c2a2f4aa4a1aba4307ed58515 2024-11-11T20:40:36,354 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/e8559155c9454635b6ff4015bc7f7d65 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e8559155c9454635b6ff4015bc7f7d65 2024-11-11T20:40:36,359 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/C of 3c30a36b246285db6085785ae9fb5d59 into dd206e7c2a2f4aa4a1aba4307ed58515(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:36,359 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:36,359 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/C, priority=11, startTime=1731357635483; duration=0sec 2024-11-11T20:40:36,359 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:36,359 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:C 2024-11-11T20:40:36,360 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3c30a36b246285db6085785ae9fb5d59/A of 3c30a36b246285db6085785ae9fb5d59 into e8559155c9454635b6ff4015bc7f7d65(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:36,360 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c30a36b246285db6085785ae9fb5d59: 2024-11-11T20:40:36,360 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., storeName=3c30a36b246285db6085785ae9fb5d59/A, priority=11, startTime=1731357635483; duration=0sec 2024-11-11T20:40:36,360 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:36,360 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c30a36b246285db6085785ae9fb5d59:A 2024-11-11T20:40:37,153 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-11T20:40:37,153 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:37,153 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:37,154 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:37,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:37,195 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-11T20:40:37,195 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:40:37,195 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:37,195 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:37,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 234 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3142 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3013 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:40:37,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1376 2024-11-11T20:40:37,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4127 rows 2024-11-11T20:40:37,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1373 2024-11-11T20:40:37,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4116 rows 2024-11-11T20:40:37,197 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:40:37,197 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:40:37,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:37,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:37,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:40:37,200 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:40:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:40:37,204 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357637204"}]},"ts":"1731357637204"} 2024-11-11T20:40:37,205 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:40:37,206 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:40:37,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:40:37,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, UNASSIGN}] 2024-11-11T20:40:37,209 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, UNASSIGN 2024-11-11T20:40:37,210 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=3c30a36b246285db6085785ae9fb5d59, regionState=CLOSING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:37,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, UNASSIGN because future has completed 2024-11-11T20:40:37,211 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:40:37,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:40:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:40:37,365 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:37,365 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:40:37,365 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 3c30a36b246285db6085785ae9fb5d59, disabling compactions & flushes 2024-11-11T20:40:37,365 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:37,365 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:37,365 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. after waiting 0 ms 2024-11-11T20:40:37,365 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:37,365 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 3c30a36b246285db6085785ae9fb5d59 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T20:40:37,371 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111b966abaafc14db69ca80e8d1dd973a9_3c30a36b246285db6085785ae9fb5d59 is 50, key is test_row_0/A:col10/1731357637192/Put/seqid=0 2024-11-11T20:40:37,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742087_1263 (size=12454) 2024-11-11T20:40:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:40:37,775 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:37,779 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111b966abaafc14db69ca80e8d1dd973a9_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111b966abaafc14db69ca80e8d1dd973a9_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:37,780 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b4777376426e47c3af3db6ac97c0b77a, store: [table=TestAcidGuarantees family=A region=3c30a36b246285db6085785ae9fb5d59] 2024-11-11T20:40:37,780 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b4777376426e47c3af3db6ac97c0b77a is 175, key is test_row_0/A:col10/1731357637192/Put/seqid=0 2024-11-11T20:40:37,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742088_1264 (size=31255) 2024-11-11T20:40:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:40:38,185 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=494, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b4777376426e47c3af3db6ac97c0b77a 2024-11-11T20:40:38,195 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/d1eaf928fe5c4bf490e5705b35bd4aad is 50, key is test_row_0/B:col10/1731357637192/Put/seqid=0 2024-11-11T20:40:38,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742089_1265 (size=12301) 2024-11-11T20:40:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:40:38,600 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/d1eaf928fe5c4bf490e5705b35bd4aad 2024-11-11T20:40:38,607 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/c40221add2874676a9356e8d8e46eb42 is 50, key is test_row_0/C:col10/1731357637192/Put/seqid=0 2024-11-11T20:40:38,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742090_1266 (size=12301) 2024-11-11T20:40:39,013 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/c40221add2874676a9356e8d8e46eb42 2024-11-11T20:40:39,024 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/A/b4777376426e47c3af3db6ac97c0b77a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b4777376426e47c3af3db6ac97c0b77a 2024-11-11T20:40:39,029 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b4777376426e47c3af3db6ac97c0b77a, entries=150, sequenceid=494, filesize=30.5 K 2024-11-11T20:40:39,030 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/B/d1eaf928fe5c4bf490e5705b35bd4aad as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/d1eaf928fe5c4bf490e5705b35bd4aad 2024-11-11T20:40:39,033 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/d1eaf928fe5c4bf490e5705b35bd4aad, entries=150, sequenceid=494, filesize=12.0 K 2024-11-11T20:40:39,034 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/.tmp/C/c40221add2874676a9356e8d8e46eb42 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c40221add2874676a9356e8d8e46eb42 2024-11-11T20:40:39,039 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c40221add2874676a9356e8d8e46eb42, entries=150, sequenceid=494, filesize=12.0 K 2024-11-11T20:40:39,040 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 3c30a36b246285db6085785ae9fb5d59 in 1675ms, sequenceid=494, compaction requested=false 2024-11-11T20:40:39,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/68ba1e1967124b169338ee8799a835ff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/930fcf4fed594c08a2524ed8ad2fd95a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/987f6e75deb74048b5383af7d967e7af, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b70cf6f663a94173b55d29be21d50c0c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/6f324b862f4b4db8bd69f3bf0e704c37, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/879f6b28b4054aee88c62163acb2bb5d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/55252b94b52143738d309fe9f4fd5d02, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/bb239513bffa4a168b49ca67f208f838, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/256252e9a35c4de0a9aeb5dec4b6d091, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ef9da3d86e244e479bdc846601530619, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f8fa505f9f1d40dab1a067792d4b0a66, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22d7dde9161640e7972db1274f9371a9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e985ce1e00424288a255bcc621289b75, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/34708b4f47a640d09d34c9e3642183cd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/4a38d0cbab334e3c81c37401e8116d51, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a76f60ea073242ec96cc6b27ab8b5ce5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/357a5a137ee2478cbaa889929aaf665d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/d5cfad00901148338d98bcda0238b85e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/840e30c2da9a4b659afbd67c462ebe7a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22002402b612457787c51b3da75cbae7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ab15c9b655504d19b36bdeeaa1c22406, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f81ed4b1ae644464a270c9c392d0335f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/5d12aa51897d4d74b1adeab63254a065, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f86a28f9d2c94fbdb6350e275ea552cf, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/45f8f9ad592643f190254585afd66cd7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/15d314e7b1514157a3a387c6695f2ce3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a07d9cd017db4a7fb5899acd825956d9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/358d16cca5f34915a1bba1e952a05db3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/7908ed6ae49c4586809eb5e8dfe7cd71, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/93602c877ca943d28f646fd45318f728, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/fafab992e51146d5bcd68b65df067098, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/9cc493101ac94a9ebdbf4ab2f81a3e37, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b348206556dd46e888bc838760bb346a] to archive 2024-11-11T20:40:39,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:40:39,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/68ba1e1967124b169338ee8799a835ff to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/68ba1e1967124b169338ee8799a835ff 2024-11-11T20:40:39,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/930fcf4fed594c08a2524ed8ad2fd95a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/930fcf4fed594c08a2524ed8ad2fd95a 2024-11-11T20:40:39,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/987f6e75deb74048b5383af7d967e7af to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/987f6e75deb74048b5383af7d967e7af 2024-11-11T20:40:39,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b70cf6f663a94173b55d29be21d50c0c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b70cf6f663a94173b55d29be21d50c0c 2024-11-11T20:40:39,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/6f324b862f4b4db8bd69f3bf0e704c37 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/6f324b862f4b4db8bd69f3bf0e704c37 2024-11-11T20:40:39,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/879f6b28b4054aee88c62163acb2bb5d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/879f6b28b4054aee88c62163acb2bb5d 2024-11-11T20:40:39,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/55252b94b52143738d309fe9f4fd5d02 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/55252b94b52143738d309fe9f4fd5d02 2024-11-11T20:40:39,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/bb239513bffa4a168b49ca67f208f838 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/bb239513bffa4a168b49ca67f208f838 2024-11-11T20:40:39,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/256252e9a35c4de0a9aeb5dec4b6d091 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/256252e9a35c4de0a9aeb5dec4b6d091 2024-11-11T20:40:39,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ef9da3d86e244e479bdc846601530619 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ef9da3d86e244e479bdc846601530619 2024-11-11T20:40:39,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f8fa505f9f1d40dab1a067792d4b0a66 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f8fa505f9f1d40dab1a067792d4b0a66 2024-11-11T20:40:39,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22d7dde9161640e7972db1274f9371a9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22d7dde9161640e7972db1274f9371a9 2024-11-11T20:40:39,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e985ce1e00424288a255bcc621289b75 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e985ce1e00424288a255bcc621289b75 2024-11-11T20:40:39,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/34708b4f47a640d09d34c9e3642183cd to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/34708b4f47a640d09d34c9e3642183cd 2024-11-11T20:40:39,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/4a38d0cbab334e3c81c37401e8116d51 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/4a38d0cbab334e3c81c37401e8116d51 2024-11-11T20:40:39,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a76f60ea073242ec96cc6b27ab8b5ce5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a76f60ea073242ec96cc6b27ab8b5ce5 2024-11-11T20:40:39,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/357a5a137ee2478cbaa889929aaf665d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/357a5a137ee2478cbaa889929aaf665d 2024-11-11T20:40:39,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/d5cfad00901148338d98bcda0238b85e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/d5cfad00901148338d98bcda0238b85e 2024-11-11T20:40:39,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/840e30c2da9a4b659afbd67c462ebe7a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/840e30c2da9a4b659afbd67c462ebe7a 2024-11-11T20:40:39,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22002402b612457787c51b3da75cbae7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/22002402b612457787c51b3da75cbae7 2024-11-11T20:40:39,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ab15c9b655504d19b36bdeeaa1c22406 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/ab15c9b655504d19b36bdeeaa1c22406 2024-11-11T20:40:39,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f81ed4b1ae644464a270c9c392d0335f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f81ed4b1ae644464a270c9c392d0335f 2024-11-11T20:40:39,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/5d12aa51897d4d74b1adeab63254a065 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/5d12aa51897d4d74b1adeab63254a065 2024-11-11T20:40:39,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f86a28f9d2c94fbdb6350e275ea552cf to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/f86a28f9d2c94fbdb6350e275ea552cf 2024-11-11T20:40:39,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/45f8f9ad592643f190254585afd66cd7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/45f8f9ad592643f190254585afd66cd7 2024-11-11T20:40:39,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/15d314e7b1514157a3a387c6695f2ce3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/15d314e7b1514157a3a387c6695f2ce3 2024-11-11T20:40:39,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a07d9cd017db4a7fb5899acd825956d9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/a07d9cd017db4a7fb5899acd825956d9 2024-11-11T20:40:39,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/358d16cca5f34915a1bba1e952a05db3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/358d16cca5f34915a1bba1e952a05db3 2024-11-11T20:40:39,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/7908ed6ae49c4586809eb5e8dfe7cd71 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/7908ed6ae49c4586809eb5e8dfe7cd71 2024-11-11T20:40:39,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/93602c877ca943d28f646fd45318f728 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/93602c877ca943d28f646fd45318f728 2024-11-11T20:40:39,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/fafab992e51146d5bcd68b65df067098 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/fafab992e51146d5bcd68b65df067098 2024-11-11T20:40:39,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/9cc493101ac94a9ebdbf4ab2f81a3e37 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/9cc493101ac94a9ebdbf4ab2f81a3e37 2024-11-11T20:40:39,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b348206556dd46e888bc838760bb346a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b348206556dd46e888bc838760bb346a 2024-11-11T20:40:39,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/67559b672b1b4699af6a519953ee5d63, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ca1064fb3fa54ee18d613d239348cc68, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b7d7a1d35fc14ca1998ab0e91615b7df, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/6a9bfdfe450745f098d209dc0f2c446d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fdb60ea61b7d4883994c3d8d0b47fa09, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/c578dced20324c579042babe0830bf44, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1093d37ed6714374ae3a1f6f44adabee, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2686472bef414fc58555c39d93f96491, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3761dd7c23c9496ea70529348b63fb34, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/cb2b29af1d4146dd9849186996e2ce32, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9c53d3fb1eff45f9bf755f9c0fab9a76, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ddbf54c1b5fe462f86ec414355c18344, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f4a25a80ffdb45b387dc0598ac488093, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/22c49ba7d58145549bfce5866b9c8f0a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b92af0662c05486fb6818ca268ababd8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1816ddf5409e47f0a040c8b83c032b21, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4ecd39684fe84e7e8a599dec1cb0268d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2d705324ee554807944a71379794d645, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a08d4f4e8988458095ad7ae18ae56626, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/5a8c705c9570425dab468c58f4178952, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9bdcda93145e468ab04607f07fd5b7c9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9e80cb4f59364083bb11039d16779a82, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ad89596c0eca4773a85a4aabaf40a0f7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3663d50c551249d383130e1b64acfda3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a4b7c91654434bc4a95071756edaa134, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fedc5d48b298472a8b6fac0bd7c22b6e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/212b5aecbfa247da8aa20899205f6820, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/331b7ceeb6f74ff7898a36ed780d67d8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/522cd9a00a4a43dabca8af2449678382, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/0a6fc7ae1f404410ac023b79ca2ba172, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/49a18d1be9194ff08c6f6de475a2340b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4515adfae0274642bb82e503ec4cb057, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f46dad21d99d453a9aeb3328d711661e] to archive 2024-11-11T20:40:39,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:40:39,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/67559b672b1b4699af6a519953ee5d63 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/67559b672b1b4699af6a519953ee5d63 2024-11-11T20:40:39,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ca1064fb3fa54ee18d613d239348cc68 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ca1064fb3fa54ee18d613d239348cc68 2024-11-11T20:40:39,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b7d7a1d35fc14ca1998ab0e91615b7df to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b7d7a1d35fc14ca1998ab0e91615b7df 2024-11-11T20:40:39,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/6a9bfdfe450745f098d209dc0f2c446d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/6a9bfdfe450745f098d209dc0f2c446d 2024-11-11T20:40:39,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fdb60ea61b7d4883994c3d8d0b47fa09 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fdb60ea61b7d4883994c3d8d0b47fa09 2024-11-11T20:40:39,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/c578dced20324c579042babe0830bf44 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/c578dced20324c579042babe0830bf44 2024-11-11T20:40:39,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1093d37ed6714374ae3a1f6f44adabee to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1093d37ed6714374ae3a1f6f44adabee 2024-11-11T20:40:39,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2686472bef414fc58555c39d93f96491 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2686472bef414fc58555c39d93f96491 2024-11-11T20:40:39,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3761dd7c23c9496ea70529348b63fb34 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3761dd7c23c9496ea70529348b63fb34 2024-11-11T20:40:39,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/cb2b29af1d4146dd9849186996e2ce32 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/cb2b29af1d4146dd9849186996e2ce32 2024-11-11T20:40:39,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9c53d3fb1eff45f9bf755f9c0fab9a76 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9c53d3fb1eff45f9bf755f9c0fab9a76 2024-11-11T20:40:39,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ddbf54c1b5fe462f86ec414355c18344 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ddbf54c1b5fe462f86ec414355c18344 2024-11-11T20:40:39,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f4a25a80ffdb45b387dc0598ac488093 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f4a25a80ffdb45b387dc0598ac488093 2024-11-11T20:40:39,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/22c49ba7d58145549bfce5866b9c8f0a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/22c49ba7d58145549bfce5866b9c8f0a 2024-11-11T20:40:39,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b92af0662c05486fb6818ca268ababd8 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/b92af0662c05486fb6818ca268ababd8 2024-11-11T20:40:39,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1816ddf5409e47f0a040c8b83c032b21 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/1816ddf5409e47f0a040c8b83c032b21 2024-11-11T20:40:39,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4ecd39684fe84e7e8a599dec1cb0268d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4ecd39684fe84e7e8a599dec1cb0268d 2024-11-11T20:40:39,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2d705324ee554807944a71379794d645 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/2d705324ee554807944a71379794d645 2024-11-11T20:40:39,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a08d4f4e8988458095ad7ae18ae56626 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a08d4f4e8988458095ad7ae18ae56626 2024-11-11T20:40:39,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/5a8c705c9570425dab468c58f4178952 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/5a8c705c9570425dab468c58f4178952 2024-11-11T20:40:39,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9bdcda93145e468ab04607f07fd5b7c9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9bdcda93145e468ab04607f07fd5b7c9 2024-11-11T20:40:39,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9e80cb4f59364083bb11039d16779a82 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/9e80cb4f59364083bb11039d16779a82 2024-11-11T20:40:39,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ad89596c0eca4773a85a4aabaf40a0f7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/ad89596c0eca4773a85a4aabaf40a0f7 2024-11-11T20:40:39,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3663d50c551249d383130e1b64acfda3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/3663d50c551249d383130e1b64acfda3 2024-11-11T20:40:39,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a4b7c91654434bc4a95071756edaa134 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/a4b7c91654434bc4a95071756edaa134 2024-11-11T20:40:39,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fedc5d48b298472a8b6fac0bd7c22b6e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/fedc5d48b298472a8b6fac0bd7c22b6e 2024-11-11T20:40:39,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/212b5aecbfa247da8aa20899205f6820 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/212b5aecbfa247da8aa20899205f6820 2024-11-11T20:40:39,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/331b7ceeb6f74ff7898a36ed780d67d8 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/331b7ceeb6f74ff7898a36ed780d67d8 2024-11-11T20:40:39,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/522cd9a00a4a43dabca8af2449678382 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/522cd9a00a4a43dabca8af2449678382 2024-11-11T20:40:39,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/0a6fc7ae1f404410ac023b79ca2ba172 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/0a6fc7ae1f404410ac023b79ca2ba172 2024-11-11T20:40:39,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/49a18d1be9194ff08c6f6de475a2340b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/49a18d1be9194ff08c6f6de475a2340b 2024-11-11T20:40:39,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4515adfae0274642bb82e503ec4cb057 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/4515adfae0274642bb82e503ec4cb057 2024-11-11T20:40:39,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f46dad21d99d453a9aeb3328d711661e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/f46dad21d99d453a9aeb3328d711661e 2024-11-11T20:40:39,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/260d9934c7084b5898da7233be6f91a4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e7c9b293afdf41c59c3a55d6949c30e9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/3234a3da5587434b921550651bb0fdcb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/f42b36173fb940dd82921c161d5b0f28, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1b0c81f9d4d1426cb664170eff0f9b91, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dd287844ddae4a61a59920427359b0a9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/96c773e2c1ec478895fcc53e64cff843, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1c3f6e9cbc3644a5819f8f5df7439c40, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/a6c80ceabfb140af9a374bee4913e7e2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6c3c5b8fae4d41f6a7a650be176fa502, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/73b1254f2d0f4f7180780ca9e7356030, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d2b3f2a307e344418f668de370e45b46, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c58c2927167e4307b177167b85cfed08, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e155fe1a6356456e90aea26cd8f7aeb2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e3fbc185a44e403197013c50fe020cc6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7977e3d818c84cbeabf88967970c3267, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d3351ffa32fb4c5cb7eb8fbe42bd22ae, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dfb53b677c3c43548d3767786d2ee5e0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/04d7de26407746c88a8132171e2ffeca, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7d29d93069234baabdc4c2f3e6367334, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c85408f9960e4a41b3d5d48fadc4228a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/830adb496c1d4730afda09ae061dc0c4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/738fca3607ac4bf3a25ba9423d6a9af6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/68cbaa20742a43e8a062f0eb7c866cc1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/8b8b1c11ded44accb418d56b1dd01300, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/0bcd7ef8609b4952ab47b9107676c34c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/45d1295751584418921b036975351bdc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6a73e418e980451c9c95afd56294a20d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/cb74b9ff09524423b8905c944fe7520e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e8fa73eba0c844dab34ec3295d5b993d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/be08332830494a5b98105ee8fe7fc690, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7513527a395d4533be70534c4ff72361, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6407f8b1e4074ef0b199df5fd243cd93] to archive 2024-11-11T20:40:39,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:40:39,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/260d9934c7084b5898da7233be6f91a4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/260d9934c7084b5898da7233be6f91a4 2024-11-11T20:40:39,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e7c9b293afdf41c59c3a55d6949c30e9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e7c9b293afdf41c59c3a55d6949c30e9 2024-11-11T20:40:39,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/3234a3da5587434b921550651bb0fdcb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/3234a3da5587434b921550651bb0fdcb 2024-11-11T20:40:39,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/f42b36173fb940dd82921c161d5b0f28 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/f42b36173fb940dd82921c161d5b0f28 2024-11-11T20:40:39,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1b0c81f9d4d1426cb664170eff0f9b91 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1b0c81f9d4d1426cb664170eff0f9b91 2024-11-11T20:40:39,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dd287844ddae4a61a59920427359b0a9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dd287844ddae4a61a59920427359b0a9 2024-11-11T20:40:39,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/96c773e2c1ec478895fcc53e64cff843 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/96c773e2c1ec478895fcc53e64cff843 2024-11-11T20:40:39,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1c3f6e9cbc3644a5819f8f5df7439c40 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/1c3f6e9cbc3644a5819f8f5df7439c40 2024-11-11T20:40:39,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/a6c80ceabfb140af9a374bee4913e7e2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/a6c80ceabfb140af9a374bee4913e7e2 2024-11-11T20:40:39,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6c3c5b8fae4d41f6a7a650be176fa502 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6c3c5b8fae4d41f6a7a650be176fa502 2024-11-11T20:40:39,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/73b1254f2d0f4f7180780ca9e7356030 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/73b1254f2d0f4f7180780ca9e7356030 2024-11-11T20:40:39,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d2b3f2a307e344418f668de370e45b46 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d2b3f2a307e344418f668de370e45b46 2024-11-11T20:40:39,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c58c2927167e4307b177167b85cfed08 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c58c2927167e4307b177167b85cfed08 2024-11-11T20:40:39,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e155fe1a6356456e90aea26cd8f7aeb2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e155fe1a6356456e90aea26cd8f7aeb2 2024-11-11T20:40:39,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e3fbc185a44e403197013c50fe020cc6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e3fbc185a44e403197013c50fe020cc6 2024-11-11T20:40:39,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7977e3d818c84cbeabf88967970c3267 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7977e3d818c84cbeabf88967970c3267 2024-11-11T20:40:39,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d3351ffa32fb4c5cb7eb8fbe42bd22ae to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/d3351ffa32fb4c5cb7eb8fbe42bd22ae 2024-11-11T20:40:39,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dfb53b677c3c43548d3767786d2ee5e0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dfb53b677c3c43548d3767786d2ee5e0 2024-11-11T20:40:39,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/04d7de26407746c88a8132171e2ffeca to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/04d7de26407746c88a8132171e2ffeca 2024-11-11T20:40:39,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7d29d93069234baabdc4c2f3e6367334 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7d29d93069234baabdc4c2f3e6367334 2024-11-11T20:40:39,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c85408f9960e4a41b3d5d48fadc4228a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c85408f9960e4a41b3d5d48fadc4228a 2024-11-11T20:40:39,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/830adb496c1d4730afda09ae061dc0c4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/830adb496c1d4730afda09ae061dc0c4 2024-11-11T20:40:39,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/738fca3607ac4bf3a25ba9423d6a9af6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/738fca3607ac4bf3a25ba9423d6a9af6 2024-11-11T20:40:39,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/68cbaa20742a43e8a062f0eb7c866cc1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/68cbaa20742a43e8a062f0eb7c866cc1 2024-11-11T20:40:39,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/8b8b1c11ded44accb418d56b1dd01300 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/8b8b1c11ded44accb418d56b1dd01300 2024-11-11T20:40:39,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/0bcd7ef8609b4952ab47b9107676c34c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/0bcd7ef8609b4952ab47b9107676c34c 2024-11-11T20:40:39,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/45d1295751584418921b036975351bdc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/45d1295751584418921b036975351bdc 2024-11-11T20:40:39,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6a73e418e980451c9c95afd56294a20d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6a73e418e980451c9c95afd56294a20d 2024-11-11T20:40:39,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/cb74b9ff09524423b8905c944fe7520e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/cb74b9ff09524423b8905c944fe7520e 2024-11-11T20:40:39,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e8fa73eba0c844dab34ec3295d5b993d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/e8fa73eba0c844dab34ec3295d5b993d 2024-11-11T20:40:39,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/be08332830494a5b98105ee8fe7fc690 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/be08332830494a5b98105ee8fe7fc690 2024-11-11T20:40:39,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7513527a395d4533be70534c4ff72361 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/7513527a395d4533be70534c4ff72361 2024-11-11T20:40:39,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6407f8b1e4074ef0b199df5fd243cd93 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/6407f8b1e4074ef0b199df5fd243cd93 2024-11-11T20:40:39,170 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/recovered.edits/497.seqid, newMaxSeqId=497, maxSeqId=4 2024-11-11T20:40:39,171 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. 2024-11-11T20:40:39,171 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 3c30a36b246285db6085785ae9fb5d59: Waiting for close lock at 1731357637365Running coprocessor pre-close hooks at 1731357637365Disabling compacts and flushes for region at 1731357637365Disabling writes for close at 1731357637365Obtaining lock to block concurrent updates at 1731357637365Preparing flush snapshotting stores in 3c30a36b246285db6085785ae9fb5d59 at 1731357637365Finished memstore snapshotting TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731357637365Flushing stores of TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59. at 1731357637366 (+1 ms)Flushing 3c30a36b246285db6085785ae9fb5d59/A: creating writer at 1731357637366Flushing mob file 3c30a36b246285db6085785ae9fb5d59/A: appending metadata at 1731357637371 (+5 ms)Flushing mob file 3c30a36b246285db6085785ae9fb5d59/A: closing flushed file at 1731357637371Flushing 3c30a36b246285db6085785ae9fb5d59/A: appending metadata at 1731357637780 (+409 ms)Flushing 3c30a36b246285db6085785ae9fb5d59/A: closing flushed file at 1731357637780Flushing 3c30a36b246285db6085785ae9fb5d59/B: creating writer at 1731357638192 (+412 ms)Flushing 3c30a36b246285db6085785ae9fb5d59/B: appending metadata at 1731357638195 (+3 ms)Flushing 3c30a36b246285db6085785ae9fb5d59/B: closing flushed file at 1731357638195Flushing 3c30a36b246285db6085785ae9fb5d59/C: creating writer at 1731357638604 (+409 ms)Flushing 3c30a36b246285db6085785ae9fb5d59/C: appending metadata at 1731357638607 (+3 ms)Flushing 3c30a36b246285db6085785ae9fb5d59/C: closing flushed file at 1731357638607Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@228ff73f: reopening flushed file at 1731357639022 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25202fc9: reopening flushed file at 1731357639029 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ae6d0c3: reopening flushed file at 1731357639034 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 3c30a36b246285db6085785ae9fb5d59 in 1675ms, sequenceid=494, compaction requested=false at 1731357639040 (+6 ms)Writing region close event to WAL at 1731357639167 (+127 ms)Running coprocessor post-close hooks at 1731357639170 (+3 ms)Closed at 1731357639171 (+1 ms) 2024-11-11T20:40:39,172 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,173 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=3c30a36b246285db6085785ae9fb5d59, regionState=CLOSED 2024-11-11T20:40:39,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:40:39,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T20:40:39,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 3c30a36b246285db6085785ae9fb5d59, server=51ca66f7ee3c,44271,1731357576703 in 1.9630 sec 2024-11-11T20:40:39,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-11T20:40:39,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c30a36b246285db6085785ae9fb5d59, UNASSIGN in 1.9680 sec 2024-11-11T20:40:39,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T20:40:39,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9720 sec 2024-11-11T20:40:39,180 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357639180"}]},"ts":"1731357639180"} 2024-11-11T20:40:39,181 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:40:39,181 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:40:39,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9820 sec 2024-11-11T20:40:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:40:39,361 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:39,362 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:40:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:39,364 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T20:40:39,365 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:39,367 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,370 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/recovered.edits] 2024-11-11T20:40:39,372 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b4777376426e47c3af3db6ac97c0b77a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/b4777376426e47c3af3db6ac97c0b77a 2024-11-11T20:40:39,374 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e8559155c9454635b6ff4015bc7f7d65 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/A/e8559155c9454635b6ff4015bc7f7d65 2024-11-11T20:40:39,376 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/5a8d8353605a4ed1840eeba8024b172e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/5a8d8353605a4ed1840eeba8024b172e 2024-11-11T20:40:39,378 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/d1eaf928fe5c4bf490e5705b35bd4aad to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/B/d1eaf928fe5c4bf490e5705b35bd4aad 2024-11-11T20:40:39,380 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c40221add2874676a9356e8d8e46eb42 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/c40221add2874676a9356e8d8e46eb42 2024-11-11T20:40:39,382 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dd206e7c2a2f4aa4a1aba4307ed58515 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/C/dd206e7c2a2f4aa4a1aba4307ed58515 2024-11-11T20:40:39,384 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/recovered.edits/497.seqid to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59/recovered.edits/497.seqid 2024-11-11T20:40:39,385 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,385 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:40:39,386 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:40:39,387 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T20:40:39,392 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111312aa1dcfaf46568537c9101b794ef6_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111312aa1dcfaf46568537c9101b794ef6_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,393 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111b966abaafc14db69ca80e8d1dd973a9_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111b966abaafc14db69ca80e8d1dd973a9_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,395 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b486ee4cfb74c51a229f556e3fc5fd5_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b486ee4cfb74c51a229f556e3fc5fd5_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,396 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112e810e03dee344dd8c6d1ad8b87c29f6_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112e810e03dee344dd8c6d1ad8b87c29f6_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,397 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112f71c4d4a5e5477980c4377cab76cf77_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112f71c4d4a5e5477980c4377cab76cf77_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,399 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113720f787da984925a3576761afd5a7a9_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113720f787da984925a3576761afd5a7a9_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,400 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111145ac8c588f6444e3ac52f1f513b49dc4_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111145ac8c588f6444e3ac52f1f513b49dc4_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,401 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114e04f7112d9e41d898fd1a7ee3933a1c_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114e04f7112d9e41d898fd1a7ee3933a1c_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,402 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111560a246e43394173bd26fdfa308a5ef3_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111560a246e43394173bd26fdfa308a5ef3_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,403 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d6db297ef1a4985840fa4b7ceb43dcc_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d6db297ef1a4985840fa4b7ceb43dcc_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,403 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111164659b74a06b48b49ce97b03ec488c22_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111164659b74a06b48b49ce97b03ec488c22_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,404 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118096c0cbc87b4840a3925e3da70e6efd_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118096c0cbc87b4840a3925e3da70e6efd_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,405 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ab8baf85f7eb4d00baa41af65faca5b3_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ab8baf85f7eb4d00baa41af65faca5b3_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,407 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ac19297a642f4ac18d56d04f7428fccc_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ac19297a642f4ac18d56d04f7428fccc_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,408 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aca1ec88206b40b0a1c535f85894bc70_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aca1ec88206b40b0a1c535f85894bc70_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,409 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d60872dc51a74e3498272d20f77211b0_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d60872dc51a74e3498272d20f77211b0_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,410 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d8bc0035933246838f0aa767ba72352c_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d8bc0035933246838f0aa767ba72352c_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,411 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dfb93484786547cc8d0bbb57c7c35d05_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dfb93484786547cc8d0bbb57c7c35d05_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,412 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e1199cb9061f4b84bfdbbb198d86b271_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e1199cb9061f4b84bfdbbb198d86b271_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,413 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e11ad90610ff4999b247f1cd9b264edb_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e11ad90610ff4999b247f1cd9b264edb_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,414 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e60290d1b5fc4af6988ec60b857c6e9b_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e60290d1b5fc4af6988ec60b857c6e9b_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,414 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e63e44a9b02d455eb8051f5e6856442d_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e63e44a9b02d455eb8051f5e6856442d_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,415 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111eb3ca1836ffe483489399aaa31d5cafc_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111eb3ca1836ffe483489399aaa31d5cafc_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,416 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f16f5e2f8b514cacb69a30b6411d86f9_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f16f5e2f8b514cacb69a30b6411d86f9_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,417 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f81313fb32864494954eb0a43e5fde39_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f81313fb32864494954eb0a43e5fde39_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,418 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fc953f5e1d8c49dda69a7df33a601cbb_3c30a36b246285db6085785ae9fb5d59 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fc953f5e1d8c49dda69a7df33a601cbb_3c30a36b246285db6085785ae9fb5d59 2024-11-11T20:40:39,418 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:40:39,420 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:39,423 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:40:39,424 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:40:39,425 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:39,425 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:40:39,426 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731357639425"}]},"ts":"9223372036854775807"} 2024-11-11T20:40:39,428 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:40:39,428 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3c30a36b246285db6085785ae9fb5d59, NAME => 'TestAcidGuarantees,,1731357610386.3c30a36b246285db6085785ae9fb5d59.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:40:39,428 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:40:39,428 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731357639428"}]},"ts":"9223372036854775807"} 2024-11-11T20:40:39,430 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:40:39,430 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:39,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-11T20:40:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T20:40:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:40:39,471 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:39,480 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 238) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1228990331_22 at /127.0.0.1:52348 [Waiting for operation #252] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1228990331_22 at /127.0.0.1:52462 [Waiting for operation #209] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2095157064_22 at /127.0.0.1:52584 [Waiting for operation #180] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2095157064_22 at /127.0.0.1:43646 [Waiting for operation #286] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 335) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4574 (was 4220) - AvailableMemoryMB LEAK? - 2024-11-11T20:40:39,489 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=4573 2024-11-11T20:40:39,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:40:39,490 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:40:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:40:39,492 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:40:39,492 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:39,492 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-11T20:40:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-11T20:40:39,493 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:40:39,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742091_1267 (size=959) 2024-11-11T20:40:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-11T20:40:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-11T20:40:39,900 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30 2024-11-11T20:40:39,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742092_1268 (size=53) 2024-11-11T20:40:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-11T20:40:40,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:40:40,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 22891ca6e2fbd305fc3d0297f1195419, disabling compactions & flushes 2024-11-11T20:40:40,307 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:40,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:40,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. after waiting 0 ms 2024-11-11T20:40:40,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:40,307 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:40,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 22891ca6e2fbd305fc3d0297f1195419: Waiting for close lock at 1731357640307Disabling compacts and flushes for region at 1731357640307Disabling writes for close at 1731357640307Writing region close event to WAL at 1731357640307Closed at 1731357640307 2024-11-11T20:40:40,308 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:40:40,309 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731357640309"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731357640309"}]},"ts":"1731357640309"} 2024-11-11T20:40:40,311 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:40:40,312 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:40:40,312 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357640312"}]},"ts":"1731357640312"} 2024-11-11T20:40:40,314 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:40:40,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22891ca6e2fbd305fc3d0297f1195419, ASSIGN}] 2024-11-11T20:40:40,316 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22891ca6e2fbd305fc3d0297f1195419, ASSIGN 2024-11-11T20:40:40,317 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22891ca6e2fbd305fc3d0297f1195419, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44271,1731357576703; forceNewPlan=false, retain=false 2024-11-11T20:40:40,468 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=22891ca6e2fbd305fc3d0297f1195419, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:40,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22891ca6e2fbd305fc3d0297f1195419, ASSIGN because future has completed 2024-11-11T20:40:40,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:40:40,625 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:40,625 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:40:40,626 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,626 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:40:40,626 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,626 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,628 INFO [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,629 INFO [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22891ca6e2fbd305fc3d0297f1195419 columnFamilyName A 2024-11-11T20:40:40,629 DEBUG [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:40,630 INFO [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] regionserver.HStore(327): Store=22891ca6e2fbd305fc3d0297f1195419/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:40:40,630 INFO [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-11T20:40:40,631 INFO [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22891ca6e2fbd305fc3d0297f1195419 columnFamilyName B 2024-11-11T20:40:40,631 DEBUG [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:40,632 INFO [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] regionserver.HStore(327): Store=22891ca6e2fbd305fc3d0297f1195419/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:40:40,632 INFO [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,633 INFO [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22891ca6e2fbd305fc3d0297f1195419 columnFamilyName C 2024-11-11T20:40:40,633 DEBUG [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:40:40,634 INFO [StoreOpener-22891ca6e2fbd305fc3d0297f1195419-1 {}] regionserver.HStore(327): Store=22891ca6e2fbd305fc3d0297f1195419/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:40:40,634 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,635 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,635 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,636 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,636 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,636 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:40:40,638 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,639 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:40:40,640 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 22891ca6e2fbd305fc3d0297f1195419; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69708481, jitterRate=0.03873731195926666}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:40:40,640 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:40,640 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 22891ca6e2fbd305fc3d0297f1195419: Running coprocessor pre-open hook at 1731357640626Writing region info on filesystem at 1731357640626Initializing all the Stores at 1731357640627 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357640627Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357640627Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357640627Cleaning up temporary data from old regions at 1731357640636 (+9 ms)Running coprocessor post-open hooks at 1731357640640 (+4 ms)Region opened successfully at 1731357640640 2024-11-11T20:40:40,641 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., pid=69, masterSystemTime=1731357640622 2024-11-11T20:40:40,643 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:40,643 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:40,644 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=22891ca6e2fbd305fc3d0297f1195419, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:40,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:40:40,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T20:40:40,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 in 176 msec 2024-11-11T20:40:40,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-11T20:40:40,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22891ca6e2fbd305fc3d0297f1195419, ASSIGN in 334 msec 2024-11-11T20:40:40,652 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:40:40,652 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357640652"}]},"ts":"1731357640652"} 2024-11-11T20:40:40,654 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:40:40,655 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:40:40,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-11T20:40:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-11T20:40:41,642 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:41,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25e579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,647 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f7783a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,651 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cadb4eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,653 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,656 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:40:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,35885,1731357576039 2024-11-11T20:40:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@205bd5c9 2024-11-11T20:40:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:40:41,658 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:40:41,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,660 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2691e146, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,662 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46efd7e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,664 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e3c744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aadb1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,668 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25170ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,669 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8dac24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,672 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf14705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,673 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db4cf8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,676 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dfb2a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,678 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48052aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,680 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3572111d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,681 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@264e4952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,683 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a381f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,684 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dff2ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,686 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb4c1f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e99969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,689 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d35f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,691 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246f5b72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1d0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:40:41,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:40:41,692 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:40:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:40:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:40:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a8a4f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:40:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:40:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:40:41,694 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:40:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d88df42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:40:41,695 DEBUG [Thread-1231 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,695 DEBUG [Thread-1237 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,695 DEBUG [Thread-1248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,695 DEBUG [Thread-1233 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,695 DEBUG [Thread-1239 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,695 DEBUG [Thread-1246 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,695 DEBUG [Thread-1244 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,696 DEBUG [Thread-1235 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,698 DEBUG [Thread-1242 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,699 DEBUG [Thread-1250 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:40:41,699 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:41,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,700 DEBUG [Thread-1242 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:40:41,700 DEBUG [Thread-1250 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,700 DEBUG [Thread-1242 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:40:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T20:40:41,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,702 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:40:41,703 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:41,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:40:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:40:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:41,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:41,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:40:41,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/11538f220f5042df83afbdde1e9f6f97 is 50, key is test_row_0/A:col10/1731357641712/Put/seqid=0 2024-11-11T20:40:41,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357701749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357701748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:41,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357701752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357701749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:41,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357701755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:41,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742093_1269 (size=12001) 2024-11-11T20:40:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:40:41,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:40:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:41,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:41,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357701860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:41,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357701860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357701860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357701860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:41,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357701871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:40:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:42,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:40:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357702076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357702076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357702076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357702076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357702080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:40:42,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:42,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:42,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/11538f220f5042df83afbdde1e9f6f97 2024-11-11T20:40:42,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/53d1e9c512a84e548a2ffbc2de232286 is 50, key is test_row_0/B:col10/1731357641712/Put/seqid=0 2024-11-11T20:40:42,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742094_1270 (size=12001) 2024-11-11T20:40:42,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/53d1e9c512a84e548a2ffbc2de232286 2024-11-11T20:40:42,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:40:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:42,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:40:42,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/1719ae5c7fe04b2d8c42794db709ef50 is 50, key is test_row_0/C:col10/1731357641712/Put/seqid=0 2024-11-11T20:40:42,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742095_1271 (size=12001) 2024-11-11T20:40:42,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/1719ae5c7fe04b2d8c42794db709ef50 2024-11-11T20:40:42,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/11538f220f5042df83afbdde1e9f6f97 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/11538f220f5042df83afbdde1e9f6f97 2024-11-11T20:40:42,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/11538f220f5042df83afbdde1e9f6f97, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:40:42,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/53d1e9c512a84e548a2ffbc2de232286 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/53d1e9c512a84e548a2ffbc2de232286 2024-11-11T20:40:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357702380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/53d1e9c512a84e548a2ffbc2de232286, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:40:42,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/1719ae5c7fe04b2d8c42794db709ef50 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/1719ae5c7fe04b2d8c42794db709ef50 2024-11-11T20:40:42,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357702391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357702391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357702391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357702391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/1719ae5c7fe04b2d8c42794db709ef50, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:40:42,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 22891ca6e2fbd305fc3d0297f1195419 in 683ms, sequenceid=13, compaction requested=false 2024-11-11T20:40:42,397 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T20:40:42,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:42,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:40:42,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:42,477 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:40:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/d934c1304bdb41c8a92f182bb9e6f3ad is 50, key is test_row_0/A:col10/1731357641744/Put/seqid=0 2024-11-11T20:40:42,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742096_1272 (size=12001) 2024-11-11T20:40:42,496 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/d934c1304bdb41c8a92f182bb9e6f3ad 2024-11-11T20:40:42,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/fda69e0588fa4d989bcc91aba2184b95 is 50, key is test_row_0/B:col10/1731357641744/Put/seqid=0 2024-11-11T20:40:42,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742097_1273 (size=12001) 2024-11-11T20:40:42,548 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/fda69e0588fa4d989bcc91aba2184b95 2024-11-11T20:40:42,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ab0240621cf447b781cf2c3ccc2c12b0 is 50, key is test_row_0/C:col10/1731357641744/Put/seqid=0 2024-11-11T20:40:42,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742098_1274 (size=12001) 2024-11-11T20:40:42,572 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ab0240621cf447b781cf2c3ccc2c12b0 2024-11-11T20:40:42,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/d934c1304bdb41c8a92f182bb9e6f3ad as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d934c1304bdb41c8a92f182bb9e6f3ad 2024-11-11T20:40:42,587 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d934c1304bdb41c8a92f182bb9e6f3ad, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T20:40:42,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/fda69e0588fa4d989bcc91aba2184b95 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fda69e0588fa4d989bcc91aba2184b95 2024-11-11T20:40:42,593 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fda69e0588fa4d989bcc91aba2184b95, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T20:40:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ab0240621cf447b781cf2c3ccc2c12b0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ab0240621cf447b781cf2c3ccc2c12b0 2024-11-11T20:40:42,600 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ab0240621cf447b781cf2c3ccc2c12b0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T20:40:42,600 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 22891ca6e2fbd305fc3d0297f1195419 in 123ms, sequenceid=37, compaction requested=false 2024-11-11T20:40:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T20:40:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T20:40:42,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T20:40:42,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 902 msec 2024-11-11T20:40:42,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 909 msec 2024-11-11T20:40:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:40:42,841 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:42,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T20:40:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:40:42,844 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:42,845 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:42,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:42,907 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:42,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:40:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:42,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/e327574638184729972e253b66c161b2 is 50, key is test_row_0/A:col10/1731357642906/Put/seqid=0 2024-11-11T20:40:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:40:42,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357702949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357702950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357702951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357702952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742099_1275 (size=14341) 2024-11-11T20:40:42,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357702953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:42,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/e327574638184729972e253b66c161b2 2024-11-11T20:40:42,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:42,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:42,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/5ad7a7a79d2d459891a0fa6202d4de63 is 50, key is test_row_0/B:col10/1731357642906/Put/seqid=0 2024-11-11T20:40:43,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T20:40:43,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:43,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742100_1276 (size=12001) 2024-11-11T20:40:43,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/5ad7a7a79d2d459891a0fa6202d4de63 2024-11-11T20:40:43,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/d15ad5ae63e84c5ca634acf24971bb20 is 50, key is test_row_0/C:col10/1731357642906/Put/seqid=0 2024-11-11T20:40:43,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357703061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357703061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357703061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357703062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357703062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742101_1277 (size=12001) 2024-11-11T20:40:43,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/d15ad5ae63e84c5ca634acf24971bb20 2024-11-11T20:40:43,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/e327574638184729972e253b66c161b2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e327574638184729972e253b66c161b2 2024-11-11T20:40:43,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e327574638184729972e253b66c161b2, entries=200, sequenceid=49, filesize=14.0 K 2024-11-11T20:40:43,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/5ad7a7a79d2d459891a0fa6202d4de63 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5ad7a7a79d2d459891a0fa6202d4de63 2024-11-11T20:40:43,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5ad7a7a79d2d459891a0fa6202d4de63, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:40:43,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/d15ad5ae63e84c5ca634acf24971bb20 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d15ad5ae63e84c5ca634acf24971bb20 2024-11-11T20:40:43,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d15ad5ae63e84c5ca634acf24971bb20, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:40:43,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 22891ca6e2fbd305fc3d0297f1195419 in 220ms, sequenceid=49, compaction requested=true 2024-11-11T20:40:43,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:43,127 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:43,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:43,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:43,128 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:43,129 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:43,129 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:43,129 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,129 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/11538f220f5042df83afbdde1e9f6f97, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d934c1304bdb41c8a92f182bb9e6f3ad, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e327574638184729972e253b66c161b2] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=37.4 K 2024-11-11T20:40:43,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:43,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:43,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:43,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:43,131 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 11538f220f5042df83afbdde1e9f6f97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731357641709 2024-11-11T20:40:43,133 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:43,133 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d934c1304bdb41c8a92f182bb9e6f3ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731357641744 2024-11-11T20:40:43,133 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:43,133 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,133 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/53d1e9c512a84e548a2ffbc2de232286, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fda69e0588fa4d989bcc91aba2184b95, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5ad7a7a79d2d459891a0fa6202d4de63] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=35.2 K 2024-11-11T20:40:43,134 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53d1e9c512a84e548a2ffbc2de232286, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731357641709 2024-11-11T20:40:43,134 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting e327574638184729972e253b66c161b2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357642903 2024-11-11T20:40:43,134 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fda69e0588fa4d989bcc91aba2184b95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731357641744 2024-11-11T20:40:43,135 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad7a7a79d2d459891a0fa6202d4de63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357642903 2024-11-11T20:40:43,160 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:43,160 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/3aa49e5c521e43c696f69342fe209153 is 50, key is test_row_0/A:col10/1731357642906/Put/seqid=0 2024-11-11T20:40:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:40:43,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T20:40:43,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,165 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:40:43,168 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:43,168 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/aeabc5c8314d47a5ab094db323940bc6 is 50, key is test_row_0/B:col10/1731357642906/Put/seqid=0 2024-11-11T20:40:43,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/888f240c7f774e129376c748c2910e5b is 50, key is test_row_0/A:col10/1731357642948/Put/seqid=0 2024-11-11T20:40:43,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742102_1278 (size=12104) 2024-11-11T20:40:43,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742103_1279 (size=12104) 2024-11-11T20:40:43,263 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/aeabc5c8314d47a5ab094db323940bc6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/aeabc5c8314d47a5ab094db323940bc6 2024-11-11T20:40:43,272 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into aeabc5c8314d47a5ab094db323940bc6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:43,272 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:43,272 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=13, startTime=1731357643128; duration=0sec 2024-11-11T20:40:43,273 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:43,273 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:43,273 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:43,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:43,274 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:43,274 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:43,274 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,276 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/1719ae5c7fe04b2d8c42794db709ef50, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ab0240621cf447b781cf2c3ccc2c12b0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d15ad5ae63e84c5ca634acf24971bb20] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=35.2 K 2024-11-11T20:40:43,276 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1719ae5c7fe04b2d8c42794db709ef50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731357641709 2024-11-11T20:40:43,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742104_1280 (size=12001) 2024-11-11T20:40:43,277 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab0240621cf447b781cf2c3ccc2c12b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731357641744 2024-11-11T20:40:43,277 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d15ad5ae63e84c5ca634acf24971bb20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357642903 2024-11-11T20:40:43,277 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/888f240c7f774e129376c748c2910e5b 2024-11-11T20:40:43,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/4b1099936d5842339630a0b6bb5ce0b5 is 50, key is test_row_0/B:col10/1731357642948/Put/seqid=0 2024-11-11T20:40:43,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357703286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357703289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357703291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357703291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357703302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,308 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#229 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:43,310 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/d1c01da880f749aab2c188ffaf3612a9 is 50, key is test_row_0/C:col10/1731357642906/Put/seqid=0 2024-11-11T20:40:43,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742105_1281 (size=12001) 2024-11-11T20:40:43,340 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/4b1099936d5842339630a0b6bb5ce0b5 2024-11-11T20:40:43,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742106_1282 (size=12104) 2024-11-11T20:40:43,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ade51343f1af436e8e55774a8f16d13b is 50, key is test_row_0/C:col10/1731357642948/Put/seqid=0 2024-11-11T20:40:43,384 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/d1c01da880f749aab2c188ffaf3612a9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d1c01da880f749aab2c188ffaf3612a9 2024-11-11T20:40:43,390 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into d1c01da880f749aab2c188ffaf3612a9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:43,390 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:43,390 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=13, startTime=1731357643130; duration=0sec 2024-11-11T20:40:43,390 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:43,390 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:43,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742107_1283 (size=12001) 2024-11-11T20:40:43,395 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ade51343f1af436e8e55774a8f16d13b 2024-11-11T20:40:43,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/888f240c7f774e129376c748c2910e5b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/888f240c7f774e129376c748c2910e5b 2024-11-11T20:40:43,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357703401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357703401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,411 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/888f240c7f774e129376c748c2910e5b, entries=150, sequenceid=73, filesize=11.7 K 2024-11-11T20:40:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/4b1099936d5842339630a0b6bb5ce0b5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4b1099936d5842339630a0b6bb5ce0b5 2024-11-11T20:40:43,424 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4b1099936d5842339630a0b6bb5ce0b5, entries=150, sequenceid=73, filesize=11.7 K 2024-11-11T20:40:43,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357703411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357703411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357703411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ade51343f1af436e8e55774a8f16d13b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ade51343f1af436e8e55774a8f16d13b 2024-11-11T20:40:43,441 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ade51343f1af436e8e55774a8f16d13b, entries=150, sequenceid=73, filesize=11.7 K 2024-11-11T20:40:43,442 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 22891ca6e2fbd305fc3d0297f1195419 in 277ms, sequenceid=73, compaction requested=false 2024-11-11T20:40:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T20:40:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T20:40:43,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T20:40:43,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 599 msec 2024-11-11T20:40:43,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 606 msec 2024-11-11T20:40:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:40:43,471 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:43,471 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T20:40:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T20:40:43,474 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:43,475 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:43,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T20:40:43,617 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:43,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T20:40:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:43,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T20:40:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:43,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,636 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/3aa49e5c521e43c696f69342fe209153 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/3aa49e5c521e43c696f69342fe209153 2024-11-11T20:40:43,644 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into 3aa49e5c521e43c696f69342fe209153(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:43,644 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:43,644 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=13, startTime=1731357643127; duration=0sec 2024-11-11T20:40:43,644 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:43,644 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:43,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/50a69978fc254afe9d9e698bf4feda5f is 50, key is test_row_0/A:col10/1731357643289/Put/seqid=0 2024-11-11T20:40:43,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357703660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357703662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357703664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357703668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357703671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742108_1284 (size=16681) 2024-11-11T20:40:43,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T20:40:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357703781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357703781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357703781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357703781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T20:40:43,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357703781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:43,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T20:40:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:43,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:43,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357703991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357704001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357704001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357704001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357704011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T20:40:44,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:44,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/50a69978fc254afe9d9e698bf4feda5f 2024-11-11T20:40:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T20:40:44,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/ab428dcb4daf4ed2af71c71d25ed79bd is 50, key is test_row_0/B:col10/1731357643289/Put/seqid=0 2024-11-11T20:40:44,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742109_1285 (size=12001) 2024-11-11T20:40:44,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/ab428dcb4daf4ed2af71c71d25ed79bd 2024-11-11T20:40:44,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/369d4a4c61d54f54a9677f845cb1e9a0 is 50, key is test_row_0/C:col10/1731357643289/Put/seqid=0 2024-11-11T20:40:44,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742110_1286 (size=12001) 2024-11-11T20:40:44,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T20:40:44,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:44,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357704300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357704311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357704311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357704311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357704323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T20:40:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T20:40:44,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T20:40:44,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/369d4a4c61d54f54a9677f845cb1e9a0 2024-11-11T20:40:44,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/50a69978fc254afe9d9e698bf4feda5f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/50a69978fc254afe9d9e698bf4feda5f 2024-11-11T20:40:44,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/50a69978fc254afe9d9e698bf4feda5f, entries=250, sequenceid=89, filesize=16.3 K 2024-11-11T20:40:44,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/ab428dcb4daf4ed2af71c71d25ed79bd as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ab428dcb4daf4ed2af71c71d25ed79bd 2024-11-11T20:40:44,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ab428dcb4daf4ed2af71c71d25ed79bd, entries=150, sequenceid=89, filesize=11.7 K 2024-11-11T20:40:44,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/369d4a4c61d54f54a9677f845cb1e9a0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/369d4a4c61d54f54a9677f845cb1e9a0 2024-11-11T20:40:44,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/369d4a4c61d54f54a9677f845cb1e9a0, entries=150, sequenceid=89, filesize=11.7 K 2024-11-11T20:40:44,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 22891ca6e2fbd305fc3d0297f1195419 in 1019ms, sequenceid=89, compaction requested=true 2024-11-11T20:40:44,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:44,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:44,637 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:44,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:44,638 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:44,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:44,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:44,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:44,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:44,639 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:44,639 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:44,639 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,639 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/3aa49e5c521e43c696f69342fe209153, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/888f240c7f774e129376c748c2910e5b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/50a69978fc254afe9d9e698bf4feda5f] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=39.8 K 2024-11-11T20:40:44,640 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa49e5c521e43c696f69342fe209153, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357642903 2024-11-11T20:40:44,640 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 888f240c7f774e129376c748c2910e5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731357642948 2024-11-11T20:40:44,640 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:44,640 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:44,640 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50a69978fc254afe9d9e698bf4feda5f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731357643274 2024-11-11T20:40:44,640 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,640 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/aeabc5c8314d47a5ab094db323940bc6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4b1099936d5842339630a0b6bb5ce0b5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ab428dcb4daf4ed2af71c71d25ed79bd] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=35.3 K 2024-11-11T20:40:44,642 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting aeabc5c8314d47a5ab094db323940bc6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357642903 2024-11-11T20:40:44,642 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1099936d5842339630a0b6bb5ce0b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731357642948 2024-11-11T20:40:44,643 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ab428dcb4daf4ed2af71c71d25ed79bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731357643289 2024-11-11T20:40:44,666 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:44,667 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/e0e3f61a3ad84a3685f1035dc1f4ba56 is 50, key is test_row_0/A:col10/1731357643289/Put/seqid=0 2024-11-11T20:40:44,671 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:44,672 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/dcb720b25ada4912b81a786af3332d0a is 50, key is test_row_0/B:col10/1731357643289/Put/seqid=0 2024-11-11T20:40:44,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T20:40:44,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,705 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:40:44,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742111_1287 (size=12207) 2024-11-11T20:40:44,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742112_1288 (size=12207) 2024-11-11T20:40:44,710 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:40:44,716 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/dcb720b25ada4912b81a786af3332d0a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dcb720b25ada4912b81a786af3332d0a 2024-11-11T20:40:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/8d3e2e9644644ed384ffc685160d7864 is 50, key is test_row_0/A:col10/1731357643657/Put/seqid=0 2024-11-11T20:40:44,725 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into dcb720b25ada4912b81a786af3332d0a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:44,725 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:44,725 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=13, startTime=1731357644638; duration=0sec 2024-11-11T20:40:44,725 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:44,725 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:44,725 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:44,726 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:44,726 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:44,727 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,727 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d1c01da880f749aab2c188ffaf3612a9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ade51343f1af436e8e55774a8f16d13b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/369d4a4c61d54f54a9677f845cb1e9a0] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=35.3 K 2024-11-11T20:40:44,727 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c01da880f749aab2c188ffaf3612a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357642903 2024-11-11T20:40:44,727 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ade51343f1af436e8e55774a8f16d13b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731357642948 2024-11-11T20:40:44,728 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 369d4a4c61d54f54a9677f845cb1e9a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731357643289 2024-11-11T20:40:44,743 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:44,744 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/d56312a3fc7545c29c7d4e51a3d79816 is 50, key is test_row_0/C:col10/1731357643289/Put/seqid=0 2024-11-11T20:40:44,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742113_1289 (size=12001) 2024-11-11T20:40:44,767 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/8d3e2e9644644ed384ffc685160d7864 2024-11-11T20:40:44,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/db02c4ee66b447a099939b0488edb5dc is 50, key is test_row_0/B:col10/1731357643657/Put/seqid=0 2024-11-11T20:40:44,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:44,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742114_1290 (size=12207) 2024-11-11T20:40:44,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742115_1291 (size=12001) 2024-11-11T20:40:44,839 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/db02c4ee66b447a099939b0488edb5dc 2024-11-11T20:40:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357704833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357704833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357704836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357704838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357704840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/c189e703520c454d8e2fca960f438be1 is 50, key is test_row_0/C:col10/1731357643657/Put/seqid=0 2024-11-11T20:40:44,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742116_1292 (size=12001) 2024-11-11T20:40:44,896 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/c189e703520c454d8e2fca960f438be1 2024-11-11T20:40:44,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/8d3e2e9644644ed384ffc685160d7864 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/8d3e2e9644644ed384ffc685160d7864 2024-11-11T20:40:44,913 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/8d3e2e9644644ed384ffc685160d7864, entries=150, sequenceid=112, filesize=11.7 K 2024-11-11T20:40:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/db02c4ee66b447a099939b0488edb5dc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/db02c4ee66b447a099939b0488edb5dc 2024-11-11T20:40:44,925 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/db02c4ee66b447a099939b0488edb5dc, entries=150, sequenceid=112, filesize=11.7 K 2024-11-11T20:40:44,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/c189e703520c454d8e2fca960f438be1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c189e703520c454d8e2fca960f438be1 2024-11-11T20:40:44,943 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c189e703520c454d8e2fca960f438be1, entries=150, sequenceid=112, filesize=11.7 K 2024-11-11T20:40:44,945 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 22891ca6e2fbd305fc3d0297f1195419 in 240ms, sequenceid=112, compaction requested=false 2024-11-11T20:40:44,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:44,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:44,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T20:40:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T20:40:44,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T20:40:44,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-11-11T20:40:44,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4810 sec 2024-11-11T20:40:44,958 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:44,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T20:40:44,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:44,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0c48a08a4ec14962b7f773ce681eb4f1 is 50, key is test_row_0/A:col10/1731357644957/Put/seqid=0 2024-11-11T20:40:45,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357704993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357704992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357704994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357704996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357704996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742117_1293 (size=12101) 2024-11-11T20:40:45,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0c48a08a4ec14962b7f773ce681eb4f1 2024-11-11T20:40:45,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/ea5351008ad1424e9abbf5d058dff66b is 50, key is test_row_0/B:col10/1731357644957/Put/seqid=0 2024-11-11T20:40:45,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742118_1294 (size=12101) 2024-11-11T20:40:45,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/ea5351008ad1424e9abbf5d058dff66b 2024-11-11T20:40:45,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/910eb3e706fd4ebcbdc35141f82c551d is 50, key is test_row_0/C:col10/1731357644957/Put/seqid=0 2024-11-11T20:40:45,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742119_1295 (size=12101) 2024-11-11T20:40:45,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/910eb3e706fd4ebcbdc35141f82c551d 2024-11-11T20:40:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0c48a08a4ec14962b7f773ce681eb4f1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0c48a08a4ec14962b7f773ce681eb4f1 2024-11-11T20:40:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357705111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357705111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357705111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357705111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357705111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,116 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/e0e3f61a3ad84a3685f1035dc1f4ba56 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e0e3f61a3ad84a3685f1035dc1f4ba56 2024-11-11T20:40:45,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0c48a08a4ec14962b7f773ce681eb4f1, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T20:40:45,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/ea5351008ad1424e9abbf5d058dff66b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ea5351008ad1424e9abbf5d058dff66b 2024-11-11T20:40:45,137 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into e0e3f61a3ad84a3685f1035dc1f4ba56(size=11.9 K), total size for store is 35.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:45,137 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:45,137 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=13, startTime=1731357644636; duration=0sec 2024-11-11T20:40:45,138 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:45,138 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:45,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ea5351008ad1424e9abbf5d058dff66b, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T20:40:45,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/910eb3e706fd4ebcbdc35141f82c551d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/910eb3e706fd4ebcbdc35141f82c551d 2024-11-11T20:40:45,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/910eb3e706fd4ebcbdc35141f82c551d, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T20:40:45,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 22891ca6e2fbd305fc3d0297f1195419 in 192ms, sequenceid=130, compaction requested=true 2024-11-11T20:40:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:45,150 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:45,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:45,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:45,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T20:40:45,151 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:45,152 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:45,152 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:45,152 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e0e3f61a3ad84a3685f1035dc1f4ba56, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/8d3e2e9644644ed384ffc685160d7864, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0c48a08a4ec14962b7f773ce681eb4f1] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=35.5 K 2024-11-11T20:40:45,153 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e3f61a3ad84a3685f1035dc1f4ba56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731357643289 2024-11-11T20:40:45,153 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d3e2e9644644ed384ffc685160d7864, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731357643657 2024-11-11T20:40:45,153 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c48a08a4ec14962b7f773ce681eb4f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357644956 2024-11-11T20:40:45,171 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#243 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:45,172 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/1b07b29f5cd542d3aa99013556bebeb0 is 50, key is test_row_0/A:col10/1731357644957/Put/seqid=0 2024-11-11T20:40:45,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742120_1296 (size=12409) 2024-11-11T20:40:45,226 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/d56312a3fc7545c29c7d4e51a3d79816 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d56312a3fc7545c29c7d4e51a3d79816 2024-11-11T20:40:45,233 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into d56312a3fc7545c29c7d4e51a3d79816(size=11.9 K), total size for store is 35.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:45,234 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:45,234 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=13, startTime=1731357644638; duration=0sec 2024-11-11T20:40:45,234 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T20:40:45,234 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:45,234 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:45,234 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:45,235 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:45,235 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:45,235 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:45,235 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dcb720b25ada4912b81a786af3332d0a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/db02c4ee66b447a099939b0488edb5dc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ea5351008ad1424e9abbf5d058dff66b] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=35.5 K 2024-11-11T20:40:45,236 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb720b25ada4912b81a786af3332d0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731357643289 2024-11-11T20:40:45,236 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting db02c4ee66b447a099939b0488edb5dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731357643657 2024-11-11T20:40:45,236 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ea5351008ad1424e9abbf5d058dff66b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357644956 2024-11-11T20:40:45,249 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:45,249 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/76c3314dd4ca49718060c135ea820fca is 50, key is test_row_0/B:col10/1731357644957/Put/seqid=0 2024-11-11T20:40:45,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742121_1297 (size=12409) 2024-11-11T20:40:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:45,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T20:40:45,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0ccba9d26e6d4722980eaa8f6a9a3d9e is 50, key is test_row_0/A:col10/1731357645324/Put/seqid=0 2024-11-11T20:40:45,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742122_1298 (size=12151) 2024-11-11T20:40:45,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357705344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357705345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357705351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357705351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357705351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357705460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357705460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357705461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357705461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357705472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T20:40:45,621 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:45,621 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T20:40:45,624 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T20:40:45,626 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:45,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:45,631 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/1b07b29f5cd542d3aa99013556bebeb0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1b07b29f5cd542d3aa99013556bebeb0 2024-11-11T20:40:45,636 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into 1b07b29f5cd542d3aa99013556bebeb0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:45,637 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:45,637 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=13, startTime=1731357645150; duration=0sec 2024-11-11T20:40:45,637 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:45,637 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:45,637 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:45,638 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:45,638 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:45,638 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:45,638 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d56312a3fc7545c29c7d4e51a3d79816, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c189e703520c454d8e2fca960f438be1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/910eb3e706fd4ebcbdc35141f82c551d] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=35.5 K 2024-11-11T20:40:45,639 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d56312a3fc7545c29c7d4e51a3d79816, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731357643289 2024-11-11T20:40:45,640 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c189e703520c454d8e2fca960f438be1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731357643657 2024-11-11T20:40:45,640 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 910eb3e706fd4ebcbdc35141f82c551d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357644956 2024-11-11T20:40:45,651 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:45,651 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b4005d7970324558b1ffa7c1b7606fdc is 50, key is test_row_0/C:col10/1731357644957/Put/seqid=0 2024-11-11T20:40:45,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357705671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357705671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357705671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357705672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,676 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/76c3314dd4ca49718060c135ea820fca as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/76c3314dd4ca49718060c135ea820fca 2024-11-11T20:40:45,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742123_1299 (size=12409) 2024-11-11T20:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357705681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:45,687 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into 76c3314dd4ca49718060c135ea820fca(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:45,687 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:45,688 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=13, startTime=1731357645150; duration=0sec 2024-11-11T20:40:45,690 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b4005d7970324558b1ffa7c1b7606fdc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b4005d7970324558b1ffa7c1b7606fdc 2024-11-11T20:40:45,690 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:45,690 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:45,697 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into b4005d7970324558b1ffa7c1b7606fdc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:45,698 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:45,698 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=13, startTime=1731357645151; duration=0sec 2024-11-11T20:40:45,698 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:45,698 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T20:40:45,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0ccba9d26e6d4722980eaa8f6a9a3d9e 2024-11-11T20:40:45,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/2a124d31c6374a708cf4af494f3b1c62 is 50, key is test_row_0/B:col10/1731357645324/Put/seqid=0 2024-11-11T20:40:45,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742124_1300 (size=12151) 2024-11-11T20:40:45,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/2a124d31c6374a708cf4af494f3b1c62 2024-11-11T20:40:45,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ea87db2925af483aa4e9b72daf90876c is 50, key is test_row_0/C:col10/1731357645324/Put/seqid=0 2024-11-11T20:40:45,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T20:40:45,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:45,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:45,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742125_1301 (size=12151) 2024-11-11T20:40:45,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ea87db2925af483aa4e9b72daf90876c 2024-11-11T20:40:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0ccba9d26e6d4722980eaa8f6a9a3d9e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0ccba9d26e6d4722980eaa8f6a9a3d9e 2024-11-11T20:40:45,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0ccba9d26e6d4722980eaa8f6a9a3d9e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T20:40:45,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/2a124d31c6374a708cf4af494f3b1c62 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/2a124d31c6374a708cf4af494f3b1c62 2024-11-11T20:40:45,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/2a124d31c6374a708cf4af494f3b1c62, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T20:40:45,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ea87db2925af483aa4e9b72daf90876c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ea87db2925af483aa4e9b72daf90876c 2024-11-11T20:40:45,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ea87db2925af483aa4e9b72daf90876c, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T20:40:45,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 22891ca6e2fbd305fc3d0297f1195419 in 487ms, sequenceid=155, compaction requested=false 2024-11-11T20:40:45,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:45,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T20:40:45,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:45,933 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T20:40:45,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/85db523c4e904a98b7e6fc44a3c86766 is 50, key is test_row_0/A:col10/1731357645348/Put/seqid=0 2024-11-11T20:40:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T20:40:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742126_1302 (size=12151) 2024-11-11T20:40:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:45,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:46,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357706004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357706004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357706006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357706007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357706009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357706121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357706121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357706121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357706121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357706121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T20:40:46,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357706331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357706331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357706331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357706331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357706332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,345 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/85db523c4e904a98b7e6fc44a3c86766 2024-11-11T20:40:46,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/5e88fc7bbcbc438481a62f473e9e312f is 50, key is test_row_0/B:col10/1731357645348/Put/seqid=0 2024-11-11T20:40:46,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742127_1303 (size=12151) 2024-11-11T20:40:46,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357706641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357706641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357706641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357706641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357706641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T20:40:46,765 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/5e88fc7bbcbc438481a62f473e9e312f 2024-11-11T20:40:46,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/82ee3d24f7384350a69a86ec2ab5bc38 is 50, key is test_row_0/C:col10/1731357645348/Put/seqid=0 2024-11-11T20:40:46,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742128_1304 (size=12151) 2024-11-11T20:40:46,844 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/82ee3d24f7384350a69a86ec2ab5bc38 2024-11-11T20:40:46,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/85db523c4e904a98b7e6fc44a3c86766 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/85db523c4e904a98b7e6fc44a3c86766 2024-11-11T20:40:46,859 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/85db523c4e904a98b7e6fc44a3c86766, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T20:40:46,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/5e88fc7bbcbc438481a62f473e9e312f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5e88fc7bbcbc438481a62f473e9e312f 2024-11-11T20:40:46,875 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5e88fc7bbcbc438481a62f473e9e312f, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T20:40:46,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/82ee3d24f7384350a69a86ec2ab5bc38 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/82ee3d24f7384350a69a86ec2ab5bc38 2024-11-11T20:40:46,884 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/82ee3d24f7384350a69a86ec2ab5bc38, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T20:40:46,886 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 22891ca6e2fbd305fc3d0297f1195419 in 953ms, sequenceid=173, compaction requested=true 2024-11-11T20:40:46,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:46,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:46,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T20:40:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T20:40:46,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T20:40:46,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2620 sec 2024-11-11T20:40:46,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.2710 sec 2024-11-11T20:40:47,156 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:47,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:40:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:47,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/2e18a1382b224fef9d3b1c2bdc9d306b is 50, key is test_row_0/A:col10/1731357647154/Put/seqid=0 2024-11-11T20:40:47,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357707169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357707171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357707180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357707180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357707188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742129_1305 (size=16931) 2024-11-11T20:40:47,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357707291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357707291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357707301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357707301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357707301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357707501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357707501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357707511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357707511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357707511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/2e18a1382b224fef9d3b1c2bdc9d306b 2024-11-11T20:40:47,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/fc592c98679843c48a211f6e7179cac1 is 50, key is test_row_0/B:col10/1731357647154/Put/seqid=0 2024-11-11T20:40:47,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742130_1306 (size=12151) 2024-11-11T20:40:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T20:40:47,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:47,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-11T20:40:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T20:40:47,773 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:47,774 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:47,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:47,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357707810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357707810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357707821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357707821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357707821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T20:40:47,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T20:40:47,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:47,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:47,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:47,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:47,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:48,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/fc592c98679843c48a211f6e7179cac1 2024-11-11T20:40:48,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8949a14ba1f949358d94f1a022149fb6 is 50, key is test_row_0/C:col10/1731357647154/Put/seqid=0 2024-11-11T20:40:48,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742131_1307 (size=12151) 2024-11-11T20:40:48,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8949a14ba1f949358d94f1a022149fb6 2024-11-11T20:40:48,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/2e18a1382b224fef9d3b1c2bdc9d306b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2e18a1382b224fef9d3b1c2bdc9d306b 2024-11-11T20:40:48,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2e18a1382b224fef9d3b1c2bdc9d306b, entries=250, sequenceid=197, filesize=16.5 K 2024-11-11T20:40:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/fc592c98679843c48a211f6e7179cac1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fc592c98679843c48a211f6e7179cac1 2024-11-11T20:40:48,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fc592c98679843c48a211f6e7179cac1, entries=150, sequenceid=197, filesize=11.9 K 2024-11-11T20:40:48,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8949a14ba1f949358d94f1a022149fb6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8949a14ba1f949358d94f1a022149fb6 2024-11-11T20:40:48,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8949a14ba1f949358d94f1a022149fb6, entries=150, sequenceid=197, filesize=11.9 K 2024-11-11T20:40:48,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 22891ca6e2fbd305fc3d0297f1195419 in 904ms, sequenceid=197, compaction requested=true 2024-11-11T20:40:48,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:48,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:48,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:48,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:48,060 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:48,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:48,060 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:48,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:48,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:48,062 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53642 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:48,062 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:48,062 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:48,062 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:48,062 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:48,062 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:48,062 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1b07b29f5cd542d3aa99013556bebeb0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0ccba9d26e6d4722980eaa8f6a9a3d9e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/85db523c4e904a98b7e6fc44a3c86766, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2e18a1382b224fef9d3b1c2bdc9d306b] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=52.4 K 2024-11-11T20:40:48,063 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/76c3314dd4ca49718060c135ea820fca, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/2a124d31c6374a708cf4af494f3b1c62, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5e88fc7bbcbc438481a62f473e9e312f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fc592c98679843c48a211f6e7179cac1] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=47.7 K 2024-11-11T20:40:48,063 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c3314dd4ca49718060c135ea820fca, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357644956 2024-11-11T20:40:48,063 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b07b29f5cd542d3aa99013556bebeb0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357644956 2024-11-11T20:40:48,063 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a124d31c6374a708cf4af494f3b1c62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731357644991 2024-11-11T20:40:48,063 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ccba9d26e6d4722980eaa8f6a9a3d9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731357644991 2024-11-11T20:40:48,063 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e88fc7bbcbc438481a62f473e9e312f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731357645344 2024-11-11T20:40:48,063 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85db523c4e904a98b7e6fc44a3c86766, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731357645344 2024-11-11T20:40:48,064 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fc592c98679843c48a211f6e7179cac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731357646006 2024-11-11T20:40:48,064 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e18a1382b224fef9d3b1c2bdc9d306b, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731357646006 2024-11-11T20:40:48,076 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:48,076 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:48,076 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/58a2397bbcd94c79a445acf2b6544d11 is 50, key is test_row_0/A:col10/1731357647154/Put/seqid=0 2024-11-11T20:40:48,077 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/50ef00f865b5460eac0887b7f9b33e38 is 50, key is test_row_0/B:col10/1731357647154/Put/seqid=0 2024-11-11T20:40:48,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742133_1309 (size=12595) 2024-11-11T20:40:48,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T20:40:48,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:48,087 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T20:40:48,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742132_1308 (size=12595) 2024-11-11T20:40:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T20:40:48,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/7a3ffc94ebdf4b95830c76c8c4b473ca is 50, key is test_row_0/A:col10/1731357647178/Put/seqid=0 2024-11-11T20:40:48,097 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/50ef00f865b5460eac0887b7f9b33e38 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/50ef00f865b5460eac0887b7f9b33e38 2024-11-11T20:40:48,100 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/58a2397bbcd94c79a445acf2b6544d11 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/58a2397bbcd94c79a445acf2b6544d11 2024-11-11T20:40:48,105 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into 50ef00f865b5460eac0887b7f9b33e38(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:48,105 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:48,105 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=12, startTime=1731357648060; duration=0sec 2024-11-11T20:40:48,105 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:48,105 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:48,105 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:48,108 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into 58a2397bbcd94c79a445acf2b6544d11(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:48,108 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:48,108 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=12, startTime=1731357648060; duration=0sec 2024-11-11T20:40:48,108 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:48,108 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:48,109 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:48,109 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:48,109 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:48,109 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b4005d7970324558b1ffa7c1b7606fdc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ea87db2925af483aa4e9b72daf90876c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/82ee3d24f7384350a69a86ec2ab5bc38, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8949a14ba1f949358d94f1a022149fb6] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=47.7 K 2024-11-11T20:40:48,109 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b4005d7970324558b1ffa7c1b7606fdc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731357644956 2024-11-11T20:40:48,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742134_1310 (size=12151) 2024-11-11T20:40:48,111 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ea87db2925af483aa4e9b72daf90876c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731357644991 2024-11-11T20:40:48,111 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ee3d24f7384350a69a86ec2ab5bc38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731357645344 2024-11-11T20:40:48,112 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8949a14ba1f949358d94f1a022149fb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731357646006 2024-11-11T20:40:48,123 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:48,123 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8f203dd4387544208a8b27d42568f480 is 50, key is test_row_0/C:col10/1731357647154/Put/seqid=0 2024-11-11T20:40:48,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742135_1311 (size=12595) 2024-11-11T20:40:48,139 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8f203dd4387544208a8b27d42568f480 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8f203dd4387544208a8b27d42568f480 2024-11-11T20:40:48,146 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into 8f203dd4387544208a8b27d42568f480(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:48,146 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:48,146 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=12, startTime=1731357648060; duration=0sec 2024-11-11T20:40:48,146 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:48,146 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:48,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:48,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357708346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357708347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357708348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357708350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357708350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T20:40:48,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357708460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357708461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357708461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357708461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357708461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,511 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/7a3ffc94ebdf4b95830c76c8c4b473ca 2024-11-11T20:40:48,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/eebd89a050244cd1a8a20c4d123f4d10 is 50, key is test_row_0/B:col10/1731357647178/Put/seqid=0 2024-11-11T20:40:48,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742136_1312 (size=12151) 2024-11-11T20:40:48,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357708671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357708671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357708671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357708671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357708671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T20:40:48,921 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/eebd89a050244cd1a8a20c4d123f4d10 2024-11-11T20:40:48,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/fa8d0ba375f04e158456214ba7e057c3 is 50, key is test_row_0/C:col10/1731357647178/Put/seqid=0 2024-11-11T20:40:48,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742137_1313 (size=12151) 2024-11-11T20:40:48,937 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/fa8d0ba375f04e158456214ba7e057c3 2024-11-11T20:40:48,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/7a3ffc94ebdf4b95830c76c8c4b473ca as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7a3ffc94ebdf4b95830c76c8c4b473ca 2024-11-11T20:40:48,947 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7a3ffc94ebdf4b95830c76c8c4b473ca, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T20:40:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/eebd89a050244cd1a8a20c4d123f4d10 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/eebd89a050244cd1a8a20c4d123f4d10 2024-11-11T20:40:48,953 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/eebd89a050244cd1a8a20c4d123f4d10, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T20:40:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/fa8d0ba375f04e158456214ba7e057c3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fa8d0ba375f04e158456214ba7e057c3 2024-11-11T20:40:48,958 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fa8d0ba375f04e158456214ba7e057c3, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T20:40:48,960 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 22891ca6e2fbd305fc3d0297f1195419 in 872ms, sequenceid=210, compaction requested=false 2024-11-11T20:40:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-11T20:40:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-11T20:40:48,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T20:40:48,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1870 sec 2024-11-11T20:40:48,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.1920 sec 2024-11-11T20:40:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:48,984 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:48,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T20:40:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357708992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357708992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357708993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357708993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357708995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:48,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/7fc9d788db9a44999be6aae622ba1e7c is 50, key is test_row_0/A:col10/1731357648984/Put/seqid=0 2024-11-11T20:40:49,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742138_1314 (size=14541) 2024-11-11T20:40:49,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/7fc9d788db9a44999be6aae622ba1e7c 2024-11-11T20:40:49,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/948241d031cc40f5b7ce5c538e595cd3 is 50, key is test_row_0/B:col10/1731357648984/Put/seqid=0 2024-11-11T20:40:49,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742139_1315 (size=12151) 2024-11-11T20:40:49,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/948241d031cc40f5b7ce5c538e595cd3 2024-11-11T20:40:49,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b61b7b9477274af39f56d33f145ed7e5 is 50, key is test_row_0/C:col10/1731357648984/Put/seqid=0 2024-11-11T20:40:49,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742140_1316 (size=12151) 2024-11-11T20:40:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357709101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357709101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357709101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357709101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357709101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357709311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357709311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357709311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357709311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357709312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b61b7b9477274af39f56d33f145ed7e5 2024-11-11T20:40:49,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/7fc9d788db9a44999be6aae622ba1e7c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7fc9d788db9a44999be6aae622ba1e7c 2024-11-11T20:40:49,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7fc9d788db9a44999be6aae622ba1e7c, entries=200, sequenceid=239, filesize=14.2 K 2024-11-11T20:40:49,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/948241d031cc40f5b7ce5c538e595cd3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/948241d031cc40f5b7ce5c538e595cd3 2024-11-11T20:40:49,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/948241d031cc40f5b7ce5c538e595cd3, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T20:40:49,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b61b7b9477274af39f56d33f145ed7e5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b61b7b9477274af39f56d33f145ed7e5 2024-11-11T20:40:49,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b61b7b9477274af39f56d33f145ed7e5, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T20:40:49,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 22891ca6e2fbd305fc3d0297f1195419 in 541ms, sequenceid=239, compaction requested=true 2024-11-11T20:40:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:49,525 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T20:40:49,526 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:49,526 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:49,526 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:49,527 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:49,527 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/50ef00f865b5460eac0887b7f9b33e38, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/eebd89a050244cd1a8a20c4d123f4d10, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/948241d031cc40f5b7ce5c538e595cd3] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=36.0 K 2024-11-11T20:40:49,527 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:49,527 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:49,527 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:49,527 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/58a2397bbcd94c79a445acf2b6544d11, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7a3ffc94ebdf4b95830c76c8c4b473ca, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7fc9d788db9a44999be6aae622ba1e7c] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=38.4 K 2024-11-11T20:40:49,527 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ef00f865b5460eac0887b7f9b33e38, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731357646006 2024-11-11T20:40:49,528 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58a2397bbcd94c79a445acf2b6544d11, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731357646006 2024-11-11T20:40:49,528 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting eebd89a050244cd1a8a20c4d123f4d10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731357647165 2024-11-11T20:40:49,528 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 948241d031cc40f5b7ce5c538e595cd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731357648348 2024-11-11T20:40:49,528 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a3ffc94ebdf4b95830c76c8c4b473ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731357647165 2024-11-11T20:40:49,529 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fc9d788db9a44999be6aae622ba1e7c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731357648346 2024-11-11T20:40:49,536 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:49,537 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/0e00260cba5842548d7e35eb0faf5b94 is 50, key is test_row_0/B:col10/1731357648984/Put/seqid=0 2024-11-11T20:40:49,539 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:49,540 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/a6a53caab4bd43fd848e0a31dec8a2eb is 50, key is test_row_0/A:col10/1731357648984/Put/seqid=0 2024-11-11T20:40:49,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742141_1317 (size=12697) 2024-11-11T20:40:49,547 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/0e00260cba5842548d7e35eb0faf5b94 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0e00260cba5842548d7e35eb0faf5b94 2024-11-11T20:40:49,552 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into 0e00260cba5842548d7e35eb0faf5b94(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:49,552 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:49,552 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=13, startTime=1731357649525; duration=0sec 2024-11-11T20:40:49,552 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:49,552 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:49,552 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:49,554 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:49,554 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:49,554 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:49,554 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8f203dd4387544208a8b27d42568f480, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fa8d0ba375f04e158456214ba7e057c3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b61b7b9477274af39f56d33f145ed7e5] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=36.0 K 2024-11-11T20:40:49,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742142_1318 (size=12697) 2024-11-11T20:40:49,555 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f203dd4387544208a8b27d42568f480, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731357646006 2024-11-11T20:40:49,555 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fa8d0ba375f04e158456214ba7e057c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731357647165 2024-11-11T20:40:49,556 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b61b7b9477274af39f56d33f145ed7e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731357648348 2024-11-11T20:40:49,563 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:49,563 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/6f87fdcdfa44430ab82ccc99b9ee1664 is 50, key is test_row_0/C:col10/1731357648984/Put/seqid=0 2024-11-11T20:40:49,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742143_1319 (size=12697) 2024-11-11T20:40:49,582 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/6f87fdcdfa44430ab82ccc99b9ee1664 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/6f87fdcdfa44430ab82ccc99b9ee1664 2024-11-11T20:40:49,588 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into 6f87fdcdfa44430ab82ccc99b9ee1664(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:49,588 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:49,589 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=13, startTime=1731357649525; duration=0sec 2024-11-11T20:40:49,589 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:49,590 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:49,622 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:49,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:40:49,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/6c15e6b440a64b6c83104096caf845e9 is 50, key is test_row_0/A:col10/1731357648985/Put/seqid=0 2024-11-11T20:40:49,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742144_1320 (size=14541) 2024-11-11T20:40:49,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/6c15e6b440a64b6c83104096caf845e9 2024-11-11T20:40:49,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/3660274e6b6b4bdebbb1eedc3bbe2cbb is 50, key is test_row_0/B:col10/1731357648985/Put/seqid=0 2024-11-11T20:40:49,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357709650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357709649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357709652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357709658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357709658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742145_1321 (size=12151) 2024-11-11T20:40:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357709760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357709760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357709760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357709771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357709771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T20:40:49,931 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:49,931 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-11T20:40:49,933 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T20:40:49,934 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:49,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:49,959 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/a6a53caab4bd43fd848e0a31dec8a2eb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/a6a53caab4bd43fd848e0a31dec8a2eb 2024-11-11T20:40:49,963 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into a6a53caab4bd43fd848e0a31dec8a2eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:49,963 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:49,963 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=13, startTime=1731357649525; duration=0sec 2024-11-11T20:40:49,963 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:49,963 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357709970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357709971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357709971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357709981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357709981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T20:40:50,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/3660274e6b6b4bdebbb1eedc3bbe2cbb 2024-11-11T20:40:50,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/fc7a699671a1455d8a5b6e3f96e043f6 is 50, key is test_row_0/C:col10/1731357648985/Put/seqid=0 2024-11-11T20:40:50,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742146_1322 (size=12151) 2024-11-11T20:40:50,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T20:40:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:50,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:50,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T20:40:50,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:50,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:50,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T20:40:50,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357710280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357710281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357710281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357710291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357710291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T20:40:50,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:50,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:50,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:50,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:50,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:50,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:50,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/fc7a699671a1455d8a5b6e3f96e043f6 2024-11-11T20:40:50,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/6c15e6b440a64b6c83104096caf845e9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6c15e6b440a64b6c83104096caf845e9 2024-11-11T20:40:50,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6c15e6b440a64b6c83104096caf845e9, entries=200, sequenceid=254, filesize=14.2 K 2024-11-11T20:40:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/3660274e6b6b4bdebbb1eedc3bbe2cbb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3660274e6b6b4bdebbb1eedc3bbe2cbb 2024-11-11T20:40:50,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3660274e6b6b4bdebbb1eedc3bbe2cbb, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T20:40:50,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/fc7a699671a1455d8a5b6e3f96e043f6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fc7a699671a1455d8a5b6e3f96e043f6 2024-11-11T20:40:50,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fc7a699671a1455d8a5b6e3f96e043f6, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T20:40:50,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 22891ca6e2fbd305fc3d0297f1195419 in 867ms, sequenceid=254, compaction requested=false 2024-11-11T20:40:50,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:50,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T20:40:50,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:50,544 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T20:40:50,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0a2d81281fe54ae18a7dd4d58ccc5e36 is 50, key is test_row_0/A:col10/1731357649645/Put/seqid=0 2024-11-11T20:40:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T20:40:50,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742147_1323 (size=12301) 2024-11-11T20:40:50,573 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0a2d81281fe54ae18a7dd4d58ccc5e36 2024-11-11T20:40:50,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/b9eaeda299d64cdd8ad8252fce1e3cdb is 50, key is test_row_0/B:col10/1731357649645/Put/seqid=0 2024-11-11T20:40:50,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742148_1324 (size=12301) 2024-11-11T20:40:50,609 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/b9eaeda299d64cdd8ad8252fce1e3cdb 2024-11-11T20:40:50,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/a1b5987f2adf4f2fb8b308918dd4e144 is 50, key is test_row_0/C:col10/1731357649645/Put/seqid=0 2024-11-11T20:40:50,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742149_1325 (size=12301) 2024-11-11T20:40:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:50,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:50,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357710810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357710814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357710814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357710815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357710821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357710931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357710931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357710931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357710931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357710931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,023 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/a1b5987f2adf4f2fb8b308918dd4e144 2024-11-11T20:40:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0a2d81281fe54ae18a7dd4d58ccc5e36 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0a2d81281fe54ae18a7dd4d58ccc5e36 2024-11-11T20:40:51,033 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0a2d81281fe54ae18a7dd4d58ccc5e36, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T20:40:51,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/b9eaeda299d64cdd8ad8252fce1e3cdb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/b9eaeda299d64cdd8ad8252fce1e3cdb 2024-11-11T20:40:51,038 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/b9eaeda299d64cdd8ad8252fce1e3cdb, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T20:40:51,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/a1b5987f2adf4f2fb8b308918dd4e144 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a1b5987f2adf4f2fb8b308918dd4e144 2024-11-11T20:40:51,046 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a1b5987f2adf4f2fb8b308918dd4e144, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T20:40:51,047 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 22891ca6e2fbd305fc3d0297f1195419 in 503ms, sequenceid=278, compaction requested=true 2024-11-11T20:40:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-11T20:40:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-11T20:40:51,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T20:40:51,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1150 sec 2024-11-11T20:40:51,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.1200 sec 2024-11-11T20:40:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T20:40:51,071 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:51,072 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-11T20:40:51,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:51,075 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T20:40:51,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:51,144 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:51,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T20:40:51,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/84214aa3a14d418baba4357b3f8ace10 is 50, key is test_row_0/A:col10/1731357651143/Put/seqid=0 2024-11-11T20:40:51,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742150_1326 (size=12301) 2024-11-11T20:40:51,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357711167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357711170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357711168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357711171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357711172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T20:40:51,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T20:40:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:51,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357711281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357711281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357711281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357711281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357711281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T20:40:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T20:40:51,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357711491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357711491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357711491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357711491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357711491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T20:40:51,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:51,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/84214aa3a14d418baba4357b3f8ace10 2024-11-11T20:40:51,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/252870b393b04af29b49a80cdfc522fc is 50, key is test_row_0/B:col10/1731357651143/Put/seqid=0 2024-11-11T20:40:51,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742151_1327 (size=12301) 2024-11-11T20:40:51,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T20:40:51,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:51,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T20:40:51,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357711801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357711801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357711801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357711801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357711810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:51,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T20:40:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T20:40:51,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:51,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:51,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:52,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/252870b393b04af29b49a80cdfc522fc 2024-11-11T20:40:52,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b9172f42fd76432f88217a3896eb79ef is 50, key is test_row_0/C:col10/1731357651143/Put/seqid=0 2024-11-11T20:40:52,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742152_1328 (size=12301) 2024-11-11T20:40:52,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T20:40:52,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:52,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:52,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:52,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:52,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T20:40:52,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T20:40:52,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:52,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:52,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:52,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357712311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357712311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357712311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357712311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:52,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357712321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:52,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b9172f42fd76432f88217a3896eb79ef 2024-11-11T20:40:52,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/84214aa3a14d418baba4357b3f8ace10 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/84214aa3a14d418baba4357b3f8ace10 2024-11-11T20:40:52,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/84214aa3a14d418baba4357b3f8ace10, entries=150, sequenceid=293, filesize=12.0 K 2024-11-11T20:40:52,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/252870b393b04af29b49a80cdfc522fc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/252870b393b04af29b49a80cdfc522fc 2024-11-11T20:40:52,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/252870b393b04af29b49a80cdfc522fc, entries=150, sequenceid=293, filesize=12.0 K 2024-11-11T20:40:52,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b9172f42fd76432f88217a3896eb79ef as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b9172f42fd76432f88217a3896eb79ef 2024-11-11T20:40:52,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b9172f42fd76432f88217a3896eb79ef, entries=150, sequenceid=293, filesize=12.0 K 2024-11-11T20:40:52,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 22891ca6e2fbd305fc3d0297f1195419 in 1301ms, sequenceid=293, compaction requested=true 2024-11-11T20:40:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:52,446 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:52,446 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:52,448 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:52,448 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51840 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:52,448 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:52,448 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:52,448 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:52,448 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:52,448 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/a6a53caab4bd43fd848e0a31dec8a2eb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6c15e6b440a64b6c83104096caf845e9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0a2d81281fe54ae18a7dd4d58ccc5e36, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/84214aa3a14d418baba4357b3f8ace10] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=50.6 K 2024-11-11T20:40:52,448 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0e00260cba5842548d7e35eb0faf5b94, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3660274e6b6b4bdebbb1eedc3bbe2cbb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/b9eaeda299d64cdd8ad8252fce1e3cdb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/252870b393b04af29b49a80cdfc522fc] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=48.3 K 2024-11-11T20:40:52,449 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e00260cba5842548d7e35eb0faf5b94, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731357648348 2024-11-11T20:40:52,449 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6a53caab4bd43fd848e0a31dec8a2eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731357648348 2024-11-11T20:40:52,449 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c15e6b440a64b6c83104096caf845e9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731357648985 2024-11-11T20:40:52,449 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3660274e6b6b4bdebbb1eedc3bbe2cbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731357648985 2024-11-11T20:40:52,449 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a2d81281fe54ae18a7dd4d58ccc5e36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731357649645 2024-11-11T20:40:52,449 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b9eaeda299d64cdd8ad8252fce1e3cdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731357649645 2024-11-11T20:40:52,449 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84214aa3a14d418baba4357b3f8ace10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731357650805 2024-11-11T20:40:52,449 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 252870b393b04af29b49a80cdfc522fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731357650805 2024-11-11T20:40:52,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T20:40:52,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:52,454 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:40:52,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/d226242d19ab47808999d46a3057e418 is 50, key is test_row_0/A:col10/1731357651168/Put/seqid=0 2024-11-11T20:40:52,475 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:52,476 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/dd19b5290a68409c86be2dba522664c5 is 50, key is test_row_0/B:col10/1731357651143/Put/seqid=0 2024-11-11T20:40:52,476 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:52,476 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/ffae07ead4dc4c55b3f77c06201c6646 is 50, key is test_row_0/A:col10/1731357651143/Put/seqid=0 2024-11-11T20:40:52,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742155_1331 (size=12983) 2024-11-11T20:40:52,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742154_1330 (size=12983) 2024-11-11T20:40:52,509 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/dd19b5290a68409c86be2dba522664c5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dd19b5290a68409c86be2dba522664c5 2024-11-11T20:40:52,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742153_1329 (size=12301) 2024-11-11T20:40:52,511 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/d226242d19ab47808999d46a3057e418 2024-11-11T20:40:52,518 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into dd19b5290a68409c86be2dba522664c5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:52,518 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:52,519 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=12, startTime=1731357652446; duration=0sec 2024-11-11T20:40:52,519 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:52,519 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:52,519 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:40:52,521 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:40:52,522 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:52,522 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:52,522 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/6f87fdcdfa44430ab82ccc99b9ee1664, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fc7a699671a1455d8a5b6e3f96e043f6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a1b5987f2adf4f2fb8b308918dd4e144, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b9172f42fd76432f88217a3896eb79ef] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=48.3 K 2024-11-11T20:40:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/442958689209493898e830949f92ee3a is 50, key is test_row_0/B:col10/1731357651168/Put/seqid=0 2024-11-11T20:40:52,530 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f87fdcdfa44430ab82ccc99b9ee1664, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731357648348 2024-11-11T20:40:52,530 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7a699671a1455d8a5b6e3f96e043f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731357648985 2024-11-11T20:40:52,531 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a1b5987f2adf4f2fb8b308918dd4e144, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731357649645 2024-11-11T20:40:52,533 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b9172f42fd76432f88217a3896eb79ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731357650805 2024-11-11T20:40:52,553 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:52,554 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8ea295098e1346b7b92a289dd85a03c6 is 50, key is test_row_0/C:col10/1731357651143/Put/seqid=0 2024-11-11T20:40:52,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742156_1332 (size=12301) 2024-11-11T20:40:52,581 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/442958689209493898e830949f92ee3a 2024-11-11T20:40:52,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/85527e8b024545e8a6eda5a3f3286239 is 50, key is test_row_0/C:col10/1731357651168/Put/seqid=0 2024-11-11T20:40:52,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742157_1333 (size=12983) 2024-11-11T20:40:52,643 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8ea295098e1346b7b92a289dd85a03c6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8ea295098e1346b7b92a289dd85a03c6 2024-11-11T20:40:52,652 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into 8ea295098e1346b7b92a289dd85a03c6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:52,652 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:52,652 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=12, startTime=1731357652446; duration=0sec 2024-11-11T20:40:52,652 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:52,652 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:52,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742158_1334 (size=12301) 2024-11-11T20:40:52,912 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/ffae07ead4dc4c55b3f77c06201c6646 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ffae07ead4dc4c55b3f77c06201c6646 2024-11-11T20:40:52,918 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into ffae07ead4dc4c55b3f77c06201c6646(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:52,918 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:52,918 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=12, startTime=1731357652446; duration=0sec 2024-11-11T20:40:52,918 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:52,918 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:53,055 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/85527e8b024545e8a6eda5a3f3286239 2024-11-11T20:40:53,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/d226242d19ab47808999d46a3057e418 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d226242d19ab47808999d46a3057e418 2024-11-11T20:40:53,063 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d226242d19ab47808999d46a3057e418, entries=150, sequenceid=315, filesize=12.0 K 2024-11-11T20:40:53,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/442958689209493898e830949f92ee3a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/442958689209493898e830949f92ee3a 2024-11-11T20:40:53,067 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/442958689209493898e830949f92ee3a, entries=150, sequenceid=315, filesize=12.0 K 2024-11-11T20:40:53,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/85527e8b024545e8a6eda5a3f3286239 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/85527e8b024545e8a6eda5a3f3286239 2024-11-11T20:40:53,072 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/85527e8b024545e8a6eda5a3f3286239, entries=150, sequenceid=315, filesize=12.0 K 2024-11-11T20:40:53,072 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 22891ca6e2fbd305fc3d0297f1195419 in 618ms, sequenceid=315, compaction requested=false 2024-11-11T20:40:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-11T20:40:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-11T20:40:53,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T20:40:53,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9990 sec 2024-11-11T20:40:53,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.0040 sec 2024-11-11T20:40:53,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T20:40:53,221 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:53,222 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-11T20:40:53,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T20:40:53,225 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:53,225 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:53,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T20:40:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:53,334 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:53,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:40:53,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/6ff2b89380364b5692eae446dc33c8ff is 50, key is test_row_0/A:col10/1731357653333/Put/seqid=0 2024-11-11T20:40:53,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742159_1335 (size=17181) 2024-11-11T20:40:53,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/6ff2b89380364b5692eae446dc33c8ff 2024-11-11T20:40:53,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357713363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357713363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357713368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/6cbfb1ff45dc49f0b223425ad88e1c80 is 50, key is test_row_0/B:col10/1731357653333/Put/seqid=0 2024-11-11T20:40:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357713369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T20:40:53,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357713376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:53,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:53,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742160_1336 (size=12301) 2024-11-11T20:40:53,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357713480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357713480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357713480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357713481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357713481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T20:40:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T20:40:53,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T20:40:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357713691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357713691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357713691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357713691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357713692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:53,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/6cbfb1ff45dc49f0b223425ad88e1c80 2024-11-11T20:40:53,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/3ab0d55567aa4edbb815e5975214009b is 50, key is test_row_0/C:col10/1731357653333/Put/seqid=0 2024-11-11T20:40:53,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742161_1337 (size=12301) 2024-11-11T20:40:53,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/3ab0d55567aa4edbb815e5975214009b 2024-11-11T20:40:53,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/6ff2b89380364b5692eae446dc33c8ff as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6ff2b89380364b5692eae446dc33c8ff 2024-11-11T20:40:53,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6ff2b89380364b5692eae446dc33c8ff, entries=250, sequenceid=329, filesize=16.8 K 2024-11-11T20:40:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/6cbfb1ff45dc49f0b223425ad88e1c80 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6cbfb1ff45dc49f0b223425ad88e1c80 2024-11-11T20:40:53,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6cbfb1ff45dc49f0b223425ad88e1c80, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T20:40:53,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/3ab0d55567aa4edbb815e5975214009b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/3ab0d55567aa4edbb815e5975214009b 2024-11-11T20:40:53,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/3ab0d55567aa4edbb815e5975214009b, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T20:40:53,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 22891ca6e2fbd305fc3d0297f1195419 in 494ms, sequenceid=329, compaction requested=true 2024-11-11T20:40:53,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:53,829 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:53,829 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:53,829 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:53,829 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:53,830 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:53,830 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:53,830 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,830 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,830 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dd19b5290a68409c86be2dba522664c5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/442958689209493898e830949f92ee3a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6cbfb1ff45dc49f0b223425ad88e1c80] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=36.7 K 2024-11-11T20:40:53,830 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ffae07ead4dc4c55b3f77c06201c6646, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d226242d19ab47808999d46a3057e418, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6ff2b89380364b5692eae446dc33c8ff] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=41.5 K 2024-11-11T20:40:53,830 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting dd19b5290a68409c86be2dba522664c5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731357650805 2024-11-11T20:40:53,830 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffae07ead4dc4c55b3f77c06201c6646, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731357650805 2024-11-11T20:40:53,830 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 442958689209493898e830949f92ee3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731357651161 2024-11-11T20:40:53,830 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d226242d19ab47808999d46a3057e418, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731357651161 2024-11-11T20:40:53,830 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cbfb1ff45dc49f0b223425ad88e1c80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357653331 2024-11-11T20:40:53,831 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ff2b89380364b5692eae446dc33c8ff, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357653328 2024-11-11T20:40:53,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T20:40:53,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,837 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T20:40:53,837 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:53,837 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:53,837 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0742949245f44eefb799a3bb52fb6d0b is 50, key is test_row_0/A:col10/1731357653333/Put/seqid=0 2024-11-11T20:40:53,837 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/17687023fd0a4bf98abf0b78cb76efa4 is 50, key is test_row_0/B:col10/1731357653333/Put/seqid=0 2024-11-11T20:40:53,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0eaa5e653d004a3e8680132b2dd38208 is 50, key is test_row_0/A:col10/1731357653367/Put/seqid=0 2024-11-11T20:40:53,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742163_1339 (size=13085) 2024-11-11T20:40:53,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742162_1338 (size=13085) 2024-11-11T20:40:53,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742164_1340 (size=12301) 2024-11-11T20:40:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T20:40:53,850 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0eaa5e653d004a3e8680132b2dd38208 2024-11-11T20:40:53,851 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/17687023fd0a4bf98abf0b78cb76efa4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/17687023fd0a4bf98abf0b78cb76efa4 2024-11-11T20:40:53,855 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into 17687023fd0a4bf98abf0b78cb76efa4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:53,855 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:53,855 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=13, startTime=1731357653829; duration=0sec 2024-11-11T20:40:53,855 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:53,855 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:53,855 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:53,856 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:53,856 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:53,856 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,856 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8ea295098e1346b7b92a289dd85a03c6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/85527e8b024545e8a6eda5a3f3286239, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/3ab0d55567aa4edbb815e5975214009b] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=36.7 K 2024-11-11T20:40:53,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/f83ec3941e854733883253da3a2d3343 is 50, key is test_row_0/B:col10/1731357653367/Put/seqid=0 2024-11-11T20:40:53,857 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea295098e1346b7b92a289dd85a03c6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731357650805 2024-11-11T20:40:53,857 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 85527e8b024545e8a6eda5a3f3286239, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731357651161 2024-11-11T20:40:53,857 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ab0d55567aa4edbb815e5975214009b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357653331 2024-11-11T20:40:53,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742165_1341 (size=12301) 2024-11-11T20:40:53,860 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/f83ec3941e854733883253da3a2d3343 2024-11-11T20:40:53,863 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:53,863 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/12e94d9a0e254dc39ff1cd83307153cb is 50, key is test_row_0/C:col10/1731357653333/Put/seqid=0 2024-11-11T20:40:53,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742166_1342 (size=13085) 2024-11-11T20:40:53,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/a89d1f922c4b423692cff3afb1513710 is 50, key is test_row_0/C:col10/1731357653367/Put/seqid=0 2024-11-11T20:40:53,883 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/12e94d9a0e254dc39ff1cd83307153cb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/12e94d9a0e254dc39ff1cd83307153cb 2024-11-11T20:40:53,887 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into 12e94d9a0e254dc39ff1cd83307153cb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:53,888 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:53,888 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=13, startTime=1731357653829; duration=0sec 2024-11-11T20:40:53,888 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:53,888 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:53,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742167_1343 (size=12301) 2024-11-11T20:40:53,893 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/a89d1f922c4b423692cff3afb1513710 2024-11-11T20:40:53,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0eaa5e653d004a3e8680132b2dd38208 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0eaa5e653d004a3e8680132b2dd38208 2024-11-11T20:40:53,903 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0eaa5e653d004a3e8680132b2dd38208, entries=150, sequenceid=355, filesize=12.0 K 2024-11-11T20:40:53,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/f83ec3941e854733883253da3a2d3343 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/f83ec3941e854733883253da3a2d3343 2024-11-11T20:40:53,908 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/f83ec3941e854733883253da3a2d3343, entries=150, sequenceid=355, filesize=12.0 K 2024-11-11T20:40:53,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/a89d1f922c4b423692cff3afb1513710 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a89d1f922c4b423692cff3afb1513710 2024-11-11T20:40:53,913 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a89d1f922c4b423692cff3afb1513710, entries=150, sequenceid=355, filesize=12.0 K 2024-11-11T20:40:53,914 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 22891ca6e2fbd305fc3d0297f1195419 in 77ms, sequenceid=355, compaction requested=false 2024-11-11T20:40:53,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:53,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:53,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-11T20:40:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-11T20:40:53,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T20:40:53,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 690 msec 2024-11-11T20:40:53,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 696 msec 2024-11-11T20:40:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:54,007 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:54,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:40:54,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/39185fa0b56b4c0fbf5e11e55beabb80 is 50, key is test_row_0/A:col10/1731357654007/Put/seqid=0 2024-11-11T20:40:54,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742168_1344 (size=17181) 2024-11-11T20:40:54,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357714057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357714057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357714058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357714062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357714063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357714170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357714170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357714171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357714171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357714171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,257 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/0742949245f44eefb799a3bb52fb6d0b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0742949245f44eefb799a3bb52fb6d0b 2024-11-11T20:40:54,263 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into 0742949245f44eefb799a3bb52fb6d0b(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:54,263 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:54,263 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=13, startTime=1731357653829; duration=0sec 2024-11-11T20:40:54,263 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:54,263 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T20:40:54,360 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:54,361 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-11T20:40:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T20:40:54,363 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:54,364 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:54,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:54,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357714380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357714381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357714381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357714381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357714381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/39185fa0b56b4c0fbf5e11e55beabb80 2024-11-11T20:40:54,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/66902a1076574e8eb68c818376f59dd6 is 50, key is test_row_0/B:col10/1731357654007/Put/seqid=0 2024-11-11T20:40:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742169_1345 (size=12301) 2024-11-11T20:40:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T20:40:54,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-11T20:40:54,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:54,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:54,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-11T20:40:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:54,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T20:40:54,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357714690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357714691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357714691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357714691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357714691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:54,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-11T20:40:54,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:54,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:54,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:54,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/66902a1076574e8eb68c818376f59dd6 2024-11-11T20:40:54,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/bf30752562a141aebd08e5b59d71cb20 is 50, key is test_row_0/C:col10/1731357654007/Put/seqid=0 2024-11-11T20:40:54,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742170_1346 (size=12301) 2024-11-11T20:40:54,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-11T20:40:54,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:54,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:54,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:54,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T20:40:55,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-11T20:40:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:55,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:55,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357715200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:55,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357715200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357715200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:55,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357715201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357715201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:55,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/bf30752562a141aebd08e5b59d71cb20 2024-11-11T20:40:55,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/39185fa0b56b4c0fbf5e11e55beabb80 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/39185fa0b56b4c0fbf5e11e55beabb80 2024-11-11T20:40:55,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/39185fa0b56b4c0fbf5e11e55beabb80, entries=250, sequenceid=370, filesize=16.8 K 2024-11-11T20:40:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/66902a1076574e8eb68c818376f59dd6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/66902a1076574e8eb68c818376f59dd6 2024-11-11T20:40:55,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/66902a1076574e8eb68c818376f59dd6, entries=150, sequenceid=370, filesize=12.0 K 2024-11-11T20:40:55,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/bf30752562a141aebd08e5b59d71cb20 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bf30752562a141aebd08e5b59d71cb20 2024-11-11T20:40:55,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bf30752562a141aebd08e5b59d71cb20, entries=150, sequenceid=370, filesize=12.0 K 2024-11-11T20:40:55,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 22891ca6e2fbd305fc3d0297f1195419 in 1242ms, sequenceid=370, compaction requested=true 2024-11-11T20:40:55,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:55,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:55,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:55,250 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:55,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:55,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:55,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:55,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:55,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:55,251 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:55,251 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:55,251 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:55,251 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:55,251 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:55,251 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:55,251 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0742949245f44eefb799a3bb52fb6d0b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0eaa5e653d004a3e8680132b2dd38208, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/39185fa0b56b4c0fbf5e11e55beabb80] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=41.6 K 2024-11-11T20:40:55,251 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/17687023fd0a4bf98abf0b78cb76efa4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/f83ec3941e854733883253da3a2d3343, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/66902a1076574e8eb68c818376f59dd6] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=36.8 K 2024-11-11T20:40:55,251 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0742949245f44eefb799a3bb52fb6d0b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357653331 2024-11-11T20:40:55,251 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 17687023fd0a4bf98abf0b78cb76efa4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357653331 2024-11-11T20:40:55,251 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f83ec3941e854733883253da3a2d3343, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731357653360 2024-11-11T20:40:55,251 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eaa5e653d004a3e8680132b2dd38208, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731357653360 2024-11-11T20:40:55,252 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 66902a1076574e8eb68c818376f59dd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731357654006 2024-11-11T20:40:55,252 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39185fa0b56b4c0fbf5e11e55beabb80, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731357654005 2024-11-11T20:40:55,258 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:55,258 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/679202a7eef74e6ea14e1d63a877a791 is 50, key is test_row_0/B:col10/1731357654007/Put/seqid=0 2024-11-11T20:40:55,261 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:55,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742171_1347 (size=13187) 2024-11-11T20:40:55,262 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/ed00517c999c487ab79708b331f9f333 is 50, key is test_row_0/A:col10/1731357654007/Put/seqid=0 2024-11-11T20:40:55,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742172_1348 (size=13187) 2024-11-11T20:40:55,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-11T20:40:55,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:55,280 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T20:40:55,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/fbab4fe58c6a4b1fa0b25df7f849032c is 50, key is test_row_0/A:col10/1731357654053/Put/seqid=0 2024-11-11T20:40:55,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742173_1349 (size=12301) 2024-11-11T20:40:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T20:40:55,668 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/679202a7eef74e6ea14e1d63a877a791 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/679202a7eef74e6ea14e1d63a877a791 2024-11-11T20:40:55,676 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/ed00517c999c487ab79708b331f9f333 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ed00517c999c487ab79708b331f9f333 2024-11-11T20:40:55,677 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into 679202a7eef74e6ea14e1d63a877a791(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:55,677 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:55,677 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=13, startTime=1731357655250; duration=0sec 2024-11-11T20:40:55,677 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:55,677 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:55,677 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:55,679 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:55,679 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:55,679 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:55,679 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/12e94d9a0e254dc39ff1cd83307153cb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a89d1f922c4b423692cff3afb1513710, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bf30752562a141aebd08e5b59d71cb20] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=36.8 K 2024-11-11T20:40:55,679 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e94d9a0e254dc39ff1cd83307153cb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357653331 2024-11-11T20:40:55,679 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a89d1f922c4b423692cff3afb1513710, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731357653360 2024-11-11T20:40:55,679 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting bf30752562a141aebd08e5b59d71cb20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731357654006 2024-11-11T20:40:55,682 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into ed00517c999c487ab79708b331f9f333(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:55,682 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:55,682 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=13, startTime=1731357655250; duration=0sec 2024-11-11T20:40:55,683 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:55,683 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:55,688 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:55,688 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/418b2654bb834f3d8234cd11296fe2a3 is 50, key is test_row_0/C:col10/1731357654007/Put/seqid=0 2024-11-11T20:40:55,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742174_1350 (size=13187) 2024-11-11T20:40:55,697 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/fbab4fe58c6a4b1fa0b25df7f849032c 2024-11-11T20:40:55,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/9ca244cc70654cfbb6f94e5bbe5cf5b4 is 50, key is test_row_0/B:col10/1731357654053/Put/seqid=0 2024-11-11T20:40:55,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742175_1351 (size=12301) 2024-11-11T20:40:56,099 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/418b2654bb834f3d8234cd11296fe2a3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/418b2654bb834f3d8234cd11296fe2a3 2024-11-11T20:40:56,105 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into 418b2654bb834f3d8234cd11296fe2a3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:56,105 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:56,105 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=13, startTime=1731357655250; duration=0sec 2024-11-11T20:40:56,105 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:56,105 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:56,127 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/9ca244cc70654cfbb6f94e5bbe5cf5b4 2024-11-11T20:40:56,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/5b86c37642fe4dedbe627be4264daaef is 50, key is test_row_0/C:col10/1731357654053/Put/seqid=0 2024-11-11T20:40:56,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742176_1352 (size=12301) 2024-11-11T20:40:56,147 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/5b86c37642fe4dedbe627be4264daaef 2024-11-11T20:40:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/fbab4fe58c6a4b1fa0b25df7f849032c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/fbab4fe58c6a4b1fa0b25df7f849032c 2024-11-11T20:40:56,156 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/fbab4fe58c6a4b1fa0b25df7f849032c, entries=150, sequenceid=394, filesize=12.0 K 2024-11-11T20:40:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/9ca244cc70654cfbb6f94e5bbe5cf5b4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/9ca244cc70654cfbb6f94e5bbe5cf5b4 2024-11-11T20:40:56,165 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/9ca244cc70654cfbb6f94e5bbe5cf5b4, entries=150, sequenceid=394, filesize=12.0 K 2024-11-11T20:40:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/5b86c37642fe4dedbe627be4264daaef as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/5b86c37642fe4dedbe627be4264daaef 2024-11-11T20:40:56,170 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/5b86c37642fe4dedbe627be4264daaef, entries=150, sequenceid=394, filesize=12.0 K 2024-11-11T20:40:56,171 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 22891ca6e2fbd305fc3d0297f1195419 in 891ms, sequenceid=394, compaction requested=false 2024-11-11T20:40:56,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:56,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:56,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-11T20:40:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-11T20:40:56,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-11T20:40:56,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8090 sec 2024-11-11T20:40:56,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.8150 sec 2024-11-11T20:40:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:56,219 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:56,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:40:56,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/2b3e167618bd4686b94c3346ab9425c2 is 50, key is test_row_0/A:col10/1731357656217/Put/seqid=0 2024-11-11T20:40:56,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742177_1353 (size=9857) 2024-11-11T20:40:56,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357716245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357716245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357716249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357716249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357716250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357716360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357716360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357716361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357716361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357716361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T20:40:56,520 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:56,521 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-11T20:40:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T20:40:56,522 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:56,523 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:56,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:56,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357716571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357716571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357716571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357716571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357716571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T20:40:56,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/2b3e167618bd4686b94c3346ab9425c2 2024-11-11T20:40:56,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/013f3ff86a7442ee809b5570fcbc424e is 50, key is test_row_0/B:col10/1731357656217/Put/seqid=0 2024-11-11T20:40:56,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742178_1354 (size=9857) 2024-11-11T20:40:56,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/013f3ff86a7442ee809b5570fcbc424e 2024-11-11T20:40:56,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b69046df8ceb49cdb22f90a849c4f59a is 50, key is test_row_0/C:col10/1731357656217/Put/seqid=0 2024-11-11T20:40:56,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742179_1355 (size=9857) 2024-11-11T20:40:56,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-11T20:40:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:56,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:56,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-11T20:40:56,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:56,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:56,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:56,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:56,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T20:40:56,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357716881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357716881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357716881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357716881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357716882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:56,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-11T20:40:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:56,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:57,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b69046df8ceb49cdb22f90a849c4f59a 2024-11-11T20:40:57,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/2b3e167618bd4686b94c3346ab9425c2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2b3e167618bd4686b94c3346ab9425c2 2024-11-11T20:40:57,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2b3e167618bd4686b94c3346ab9425c2, entries=100, sequenceid=409, filesize=9.6 K 2024-11-11T20:40:57,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/013f3ff86a7442ee809b5570fcbc424e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/013f3ff86a7442ee809b5570fcbc424e 2024-11-11T20:40:57,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/013f3ff86a7442ee809b5570fcbc424e, entries=100, sequenceid=409, filesize=9.6 K 2024-11-11T20:40:57,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b69046df8ceb49cdb22f90a849c4f59a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b69046df8ceb49cdb22f90a849c4f59a 2024-11-11T20:40:57,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b69046df8ceb49cdb22f90a849c4f59a, entries=100, sequenceid=409, filesize=9.6 K 2024-11-11T20:40:57,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 22891ca6e2fbd305fc3d0297f1195419 in 868ms, sequenceid=409, compaction requested=true 2024-11-11T20:40:57,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:57,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:57,088 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:57,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:57,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:57,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:57,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:57,088 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:57,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:57,094 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:57,094 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:57,094 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:57,094 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:57,094 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:57,094 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/679202a7eef74e6ea14e1d63a877a791, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/9ca244cc70654cfbb6f94e5bbe5cf5b4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/013f3ff86a7442ee809b5570fcbc424e] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=34.5 K 2024-11-11T20:40:57,095 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:57,095 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ed00517c999c487ab79708b331f9f333, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/fbab4fe58c6a4b1fa0b25df7f849032c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2b3e167618bd4686b94c3346ab9425c2] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=34.5 K 2024-11-11T20:40:57,095 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 679202a7eef74e6ea14e1d63a877a791, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731357654006 2024-11-11T20:40:57,095 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed00517c999c487ab79708b331f9f333, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731357654006 2024-11-11T20:40:57,096 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbab4fe58c6a4b1fa0b25df7f849032c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731357654047 2024-11-11T20:40:57,096 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca244cc70654cfbb6f94e5bbe5cf5b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731357654047 2024-11-11T20:40:57,096 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 013f3ff86a7442ee809b5570fcbc424e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731357656217 2024-11-11T20:40:57,096 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b3e167618bd4686b94c3346ab9425c2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731357656217 2024-11-11T20:40:57,120 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:57,121 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/1ee46546bf65422798af58e4900647f5 is 50, key is test_row_0/A:col10/1731357656217/Put/seqid=0 2024-11-11T20:40:57,126 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:57,127 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/6d040f96cb6e4d6a8175746ee468bb8a is 50, key is test_row_0/B:col10/1731357656217/Put/seqid=0 2024-11-11T20:40:57,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-11T20:40:57,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:57,137 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T20:40:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/d9e70ac682964a9eb4bec65067253273 is 50, key is test_row_0/A:col10/1731357656244/Put/seqid=0 2024-11-11T20:40:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T20:40:57,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742181_1357 (size=13289) 2024-11-11T20:40:57,174 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/6d040f96cb6e4d6a8175746ee468bb8a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6d040f96cb6e4d6a8175746ee468bb8a 2024-11-11T20:40:57,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742180_1356 (size=13289) 2024-11-11T20:40:57,189 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into 6d040f96cb6e4d6a8175746ee468bb8a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:57,189 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:57,189 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=13, startTime=1731357657088; duration=0sec 2024-11-11T20:40:57,189 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:57,189 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:57,189 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:57,192 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:57,192 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:57,192 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:57,192 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/418b2654bb834f3d8234cd11296fe2a3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/5b86c37642fe4dedbe627be4264daaef, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b69046df8ceb49cdb22f90a849c4f59a] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=34.5 K 2024-11-11T20:40:57,193 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 418b2654bb834f3d8234cd11296fe2a3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731357654006 2024-11-11T20:40:57,193 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b86c37642fe4dedbe627be4264daaef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731357654047 2024-11-11T20:40:57,194 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b69046df8ceb49cdb22f90a849c4f59a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731357656217 2024-11-11T20:40:57,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742182_1358 (size=12301) 2024-11-11T20:40:57,209 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/d9e70ac682964a9eb4bec65067253273 2024-11-11T20:40:57,214 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:57,215 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b7b3f49ee5b14bf3a63b8736d55db50a is 50, key is test_row_0/C:col10/1731357656217/Put/seqid=0 2024-11-11T20:40:57,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/6c418b0362624714963f513332b4fd88 is 50, key is test_row_0/B:col10/1731357656244/Put/seqid=0 2024-11-11T20:40:57,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742183_1359 (size=13289) 2024-11-11T20:40:57,251 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/b7b3f49ee5b14bf3a63b8736d55db50a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b7b3f49ee5b14bf3a63b8736d55db50a 2024-11-11T20:40:57,261 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into b7b3f49ee5b14bf3a63b8736d55db50a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:57,261 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:57,262 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=13, startTime=1731357657088; duration=0sec 2024-11-11T20:40:57,262 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:57,262 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:57,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742184_1360 (size=12301) 2024-11-11T20:40:57,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:57,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357717398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357717401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357717411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357717411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357717411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357717521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357717521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357717521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357717522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357717522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,588 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/1ee46546bf65422798af58e4900647f5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1ee46546bf65422798af58e4900647f5 2024-11-11T20:40:57,595 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into 1ee46546bf65422798af58e4900647f5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:57,595 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:57,595 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=13, startTime=1731357657088; duration=0sec 2024-11-11T20:40:57,595 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:57,595 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T20:40:57,672 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/6c418b0362624714963f513332b4fd88 2024-11-11T20:40:57,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8902b7d41c18413997c5f3c330569b63 is 50, key is test_row_0/C:col10/1731357656244/Put/seqid=0 2024-11-11T20:40:57,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742185_1361 (size=12301) 2024-11-11T20:40:57,724 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8902b7d41c18413997c5f3c330569b63 2024-11-11T20:40:57,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357717731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357717731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357717731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357717731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/d9e70ac682964a9eb4bec65067253273 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d9e70ac682964a9eb4bec65067253273 2024-11-11T20:40:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357717731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:57,744 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d9e70ac682964a9eb4bec65067253273, entries=150, sequenceid=435, filesize=12.0 K 2024-11-11T20:40:57,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/6c418b0362624714963f513332b4fd88 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6c418b0362624714963f513332b4fd88 2024-11-11T20:40:57,751 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6c418b0362624714963f513332b4fd88, entries=150, sequenceid=435, filesize=12.0 K 2024-11-11T20:40:57,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/8902b7d41c18413997c5f3c330569b63 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8902b7d41c18413997c5f3c330569b63 2024-11-11T20:40:57,759 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8902b7d41c18413997c5f3c330569b63, entries=150, sequenceid=435, filesize=12.0 K 2024-11-11T20:40:57,760 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 22891ca6e2fbd305fc3d0297f1195419 in 623ms, sequenceid=435, compaction requested=false 2024-11-11T20:40:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:57,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-11T20:40:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-11T20:40:57,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T20:40:57,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2390 sec 2024-11-11T20:40:57,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.2530 sec 2024-11-11T20:40:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:58,044 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:58,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T20:40:58,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/4142b69cafa340efa9906d291d76d6c5 is 50, key is test_row_0/A:col10/1731357657396/Put/seqid=0 2024-11-11T20:40:58,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742186_1362 (size=12301) 2024-11-11T20:40:58,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357718070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357718069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357718073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357718075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357718075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357718181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357718181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357718191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357718191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357718191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357718390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357718390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357718401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357718401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357718401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/4142b69cafa340efa9906d291d76d6c5 2024-11-11T20:40:58,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/0d5801718f3b4f7cb36709051514554b is 50, key is test_row_0/B:col10/1731357657396/Put/seqid=0 2024-11-11T20:40:58,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742187_1363 (size=12301) 2024-11-11T20:40:58,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/0d5801718f3b4f7cb36709051514554b 2024-11-11T20:40:58,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/bdf689268a0d4a6c882695f42ad870da is 50, key is test_row_0/C:col10/1731357657396/Put/seqid=0 2024-11-11T20:40:58,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742188_1364 (size=12301) 2024-11-11T20:40:58,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/bdf689268a0d4a6c882695f42ad870da 2024-11-11T20:40:58,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/4142b69cafa340efa9906d291d76d6c5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/4142b69cafa340efa9906d291d76d6c5 2024-11-11T20:40:58,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/4142b69cafa340efa9906d291d76d6c5, entries=150, sequenceid=453, filesize=12.0 K 2024-11-11T20:40:58,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/0d5801718f3b4f7cb36709051514554b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0d5801718f3b4f7cb36709051514554b 2024-11-11T20:40:58,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0d5801718f3b4f7cb36709051514554b, entries=150, sequenceid=453, filesize=12.0 K 2024-11-11T20:40:58,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/bdf689268a0d4a6c882695f42ad870da as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bdf689268a0d4a6c882695f42ad870da 2024-11-11T20:40:58,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bdf689268a0d4a6c882695f42ad870da, entries=150, sequenceid=453, filesize=12.0 K 2024-11-11T20:40:58,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 22891ca6e2fbd305fc3d0297f1195419 in 480ms, sequenceid=453, compaction requested=true 2024-11-11T20:40:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:40:58,524 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:40:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:40:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T20:40:58,525 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:58,526 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:58,526 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:40:58,526 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:58,526 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1ee46546bf65422798af58e4900647f5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d9e70ac682964a9eb4bec65067253273, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/4142b69cafa340efa9906d291d76d6c5] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=37.0 K 2024-11-11T20:40:58,526 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:58,526 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:40:58,526 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:58,527 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ee46546bf65422798af58e4900647f5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731357654062 2024-11-11T20:40:58,527 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6d040f96cb6e4d6a8175746ee468bb8a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6c418b0362624714963f513332b4fd88, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0d5801718f3b4f7cb36709051514554b] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=37.0 K 2024-11-11T20:40:58,527 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9e70ac682964a9eb4bec65067253273, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731357656244 2024-11-11T20:40:58,527 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d040f96cb6e4d6a8175746ee468bb8a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731357654062 2024-11-11T20:40:58,527 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c418b0362624714963f513332b4fd88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731357656244 2024-11-11T20:40:58,527 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4142b69cafa340efa9906d291d76d6c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731357657396 2024-11-11T20:40:58,528 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d5801718f3b4f7cb36709051514554b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731357657396 2024-11-11T20:40:58,547 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:58,548 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/ec73ee6b0d584d4380b7e86a8244eb9a is 50, key is test_row_0/A:col10/1731357657396/Put/seqid=0 2024-11-11T20:40:58,550 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:58,550 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/002caa58323945368bd01e8f4984e756 is 50, key is test_row_0/B:col10/1731357657396/Put/seqid=0 2024-11-11T20:40:58,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742189_1365 (size=13391) 2024-11-11T20:40:58,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742190_1366 (size=13391) 2024-11-11T20:40:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T20:40:58,671 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:40:58,672 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:40:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-11T20:40:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T20:40:58,673 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:40:58,674 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:40:58,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:40:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:58,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:40:58,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T20:40:58,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357718717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357718718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357718718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/b92949e66b3a4837a53a0d218e9d82c1 is 50, key is test_row_0/A:col10/1731357658071/Put/seqid=0 2024-11-11T20:40:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357718722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357718723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742191_1367 (size=17181) 2024-11-11T20:40:58,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/b92949e66b3a4837a53a0d218e9d82c1 2024-11-11T20:40:58,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/d87d5811313547b485e3d7ac2078ca0c is 50, key is test_row_0/B:col10/1731357658071/Put/seqid=0 2024-11-11T20:40:58,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742192_1368 (size=12301) 2024-11-11T20:40:58,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/d87d5811313547b485e3d7ac2078ca0c 2024-11-11T20:40:58,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/46c47b35944946b786ca804c3b7c7506 is 50, key is test_row_0/C:col10/1731357658071/Put/seqid=0 2024-11-11T20:40:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T20:40:58,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742193_1369 (size=12301) 2024-11-11T20:40:58,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-11T20:40:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:58,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:58,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357718830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357718831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357718831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357718831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357718831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:58,966 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/002caa58323945368bd01e8f4984e756 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/002caa58323945368bd01e8f4984e756 2024-11-11T20:40:58,977 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into 002caa58323945368bd01e8f4984e756(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:58,978 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:58,978 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=13, startTime=1731357658524; duration=0sec 2024-11-11T20:40:58,978 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:40:58,978 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:40:58,978 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:40:58,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-11T20:40:58,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:58,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:58,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:58,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:58,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:58,983 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:40:58,984 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:40:58,984 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/ec73ee6b0d584d4380b7e86a8244eb9a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ec73ee6b0d584d4380b7e86a8244eb9a 2024-11-11T20:40:58,984 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:58,984 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b7b3f49ee5b14bf3a63b8736d55db50a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8902b7d41c18413997c5f3c330569b63, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bdf689268a0d4a6c882695f42ad870da] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=37.0 K 2024-11-11T20:40:58,985 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b3f49ee5b14bf3a63b8736d55db50a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731357654062 2024-11-11T20:40:58,986 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8902b7d41c18413997c5f3c330569b63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731357656244 2024-11-11T20:40:58,986 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf689268a0d4a6c882695f42ad870da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731357657396 2024-11-11T20:40:58,990 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into ec73ee6b0d584d4380b7e86a8244eb9a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:58,990 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:58,990 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=13, startTime=1731357658524; duration=0sec 2024-11-11T20:40:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T20:40:58,990 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:58,990 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:40:58,998 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:40:58,998 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/c52dd230f96a4579bf81f59ba84cabd3 is 50, key is test_row_0/C:col10/1731357657396/Put/seqid=0 2024-11-11T20:40:59,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742194_1370 (size=13391) 2024-11-11T20:40:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357719041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357719041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357719041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357719041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357719041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-11T20:40:59,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:59,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:40:59,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/46c47b35944946b786ca804c3b7c7506 2024-11-11T20:40:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/b92949e66b3a4837a53a0d218e9d82c1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/b92949e66b3a4837a53a0d218e9d82c1 2024-11-11T20:40:59,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/b92949e66b3a4837a53a0d218e9d82c1, entries=250, sequenceid=477, filesize=16.8 K 2024-11-11T20:40:59,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/d87d5811313547b485e3d7ac2078ca0c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/d87d5811313547b485e3d7ac2078ca0c 2024-11-11T20:40:59,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/d87d5811313547b485e3d7ac2078ca0c, entries=150, sequenceid=477, filesize=12.0 K 2024-11-11T20:40:59,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/46c47b35944946b786ca804c3b7c7506 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/46c47b35944946b786ca804c3b7c7506 2024-11-11T20:40:59,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/46c47b35944946b786ca804c3b7c7506, entries=150, sequenceid=477, filesize=12.0 K 2024-11-11T20:40:59,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 22891ca6e2fbd305fc3d0297f1195419 in 501ms, sequenceid=477, compaction requested=false 2024-11-11T20:40:59,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:59,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-11T20:40:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:40:59,287 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T20:40:59,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/1572d47d1c604aae8fc10bc7f419bef5 is 50, key is test_row_0/A:col10/1731357658720/Put/seqid=0 2024-11-11T20:40:59,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742195_1371 (size=12301) 2024-11-11T20:40:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T20:40:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:40:59,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:40:59,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357719372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357719372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357719373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357719373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357719374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,410 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/c52dd230f96a4579bf81f59ba84cabd3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c52dd230f96a4579bf81f59ba84cabd3 2024-11-11T20:40:59,414 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into c52dd230f96a4579bf81f59ba84cabd3(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:40:59,414 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:40:59,414 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=13, startTime=1731357658524; duration=0sec 2024-11-11T20:40:59,414 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:40:59,414 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:40:59,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357719480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357719480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357719480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357719480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357719480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357719690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357719690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357719690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357719690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:40:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357719690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:40:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:40:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:40:59,696 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/1572d47d1c604aae8fc10bc7f419bef5 2024-11-11T20:40:59,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/8bd67c0b6ed745a091a1445063ca4c9e is 50, key is test_row_0/B:col10/1731357658720/Put/seqid=0 2024-11-11T20:40:59,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742196_1372 (size=12301) 2024-11-11T20:40:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T20:41:00,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357720000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357720000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357720000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357720001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357720001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,105 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/8bd67c0b6ed745a091a1445063ca4c9e 2024-11-11T20:41:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/e5d373ba3ec248c9b08ddd84ddcf41cf is 50, key is test_row_0/C:col10/1731357658720/Put/seqid=0 2024-11-11T20:41:00,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742197_1373 (size=12301) 2024-11-11T20:41:00,114 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/e5d373ba3ec248c9b08ddd84ddcf41cf 2024-11-11T20:41:00,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/1572d47d1c604aae8fc10bc7f419bef5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1572d47d1c604aae8fc10bc7f419bef5 2024-11-11T20:41:00,121 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1572d47d1c604aae8fc10bc7f419bef5, entries=150, sequenceid=492, filesize=12.0 K 2024-11-11T20:41:00,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/8bd67c0b6ed745a091a1445063ca4c9e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/8bd67c0b6ed745a091a1445063ca4c9e 2024-11-11T20:41:00,125 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/8bd67c0b6ed745a091a1445063ca4c9e, entries=150, sequenceid=492, filesize=12.0 K 2024-11-11T20:41:00,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/e5d373ba3ec248c9b08ddd84ddcf41cf as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/e5d373ba3ec248c9b08ddd84ddcf41cf 2024-11-11T20:41:00,129 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/e5d373ba3ec248c9b08ddd84ddcf41cf, entries=150, sequenceid=492, filesize=12.0 K 2024-11-11T20:41:00,129 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 22891ca6e2fbd305fc3d0297f1195419 in 842ms, sequenceid=492, compaction requested=true 2024-11-11T20:41:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:41:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-11T20:41:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-11T20:41:00,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-11T20:41:00,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-11-11T20:41:00,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.4600 sec 2024-11-11T20:41:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:41:00,513 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:00,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:41:00,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/dd5e1655f2a54b38acc1e123bad16e61 is 50, key is test_row_0/A:col10/1731357659373/Put/seqid=0 2024-11-11T20:41:00,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742198_1374 (size=17181) 2024-11-11T20:41:00,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357720521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357720521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357720521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357720522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357720522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357720631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357720631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357720631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357720631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357720631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T20:41:00,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:00,832 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees 2024-11-11T20:41:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T20:41:00,833 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:00,833 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:00,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:00,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357720841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357720841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357720841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357720841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357720841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:00,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/dd5e1655f2a54b38acc1e123bad16e61 2024-11-11T20:41:00,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/3ee7a872b55b4c0793fac7e02f034a87 is 50, key is test_row_0/B:col10/1731357659373/Put/seqid=0 2024-11-11T20:41:00,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742199_1375 (size=12301) 2024-11-11T20:41:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T20:41:00,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-11T20:41:00,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:00,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:41:00,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:00,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:00,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-11T20:41:01,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:41:01,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T20:41:01,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357721151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357721151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357721151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357721151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357721151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-11T20:41:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:41:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/3ee7a872b55b4c0793fac7e02f034a87 2024-11-11T20:41:01,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ebdaf94e46cf4648b76714405b522625 is 50, key is test_row_0/C:col10/1731357659373/Put/seqid=0 2024-11-11T20:41:01,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742200_1376 (size=12301) 2024-11-11T20:41:01,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-11T20:41:01,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:41:01,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T20:41:01,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-11T20:41:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:41:01,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50070 deadline: 1731357721661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1731357721661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50086 deadline: 1731357721661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50112 deadline: 1731357721661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50094 deadline: 1731357721661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:01,699 INFO [Thread-1242 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1242. 2024-11-11T20:41:01,699 INFO [Thread-1244 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1244. 2024-11-11T20:41:01,700 DEBUG [Thread-1242 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:01,700 DEBUG [Thread-1244 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:01,700 DEBUG [Thread-1242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,700 DEBUG [Thread-1244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,700 DEBUG [Thread-1242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:01,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:01,700 DEBUG [Thread-1244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,701 INFO [Thread-1248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1248. 2024-11-11T20:41:01,701 DEBUG [Thread-1248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:01,701 DEBUG [Thread-1248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,701 DEBUG [Thread-1248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:01,703 INFO [Thread-1250 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1250. 2024-11-11T20:41:01,703 DEBUG [Thread-1250 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:01,703 DEBUG [Thread-1250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,703 DEBUG [Thread-1250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:01,703 INFO [Thread-1246 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1246. 2024-11-11T20:41:01,703 DEBUG [Thread-1246 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:01,703 DEBUG [Thread-1246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,703 DEBUG [Thread-1246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:01,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:01,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ebdaf94e46cf4648b76714405b522625 2024-11-11T20:41:01,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-11T20:41:01,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:41:01,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:01,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/dd5e1655f2a54b38acc1e123bad16e61 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/dd5e1655f2a54b38acc1e123bad16e61 2024-11-11T20:41:01,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/dd5e1655f2a54b38acc1e123bad16e61, entries=250, sequenceid=518, filesize=16.8 K 2024-11-11T20:41:01,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/3ee7a872b55b4c0793fac7e02f034a87 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3ee7a872b55b4c0793fac7e02f034a87 2024-11-11T20:41:01,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3ee7a872b55b4c0793fac7e02f034a87, entries=150, sequenceid=518, filesize=12.0 K 2024-11-11T20:41:01,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/ebdaf94e46cf4648b76714405b522625 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ebdaf94e46cf4648b76714405b522625 2024-11-11T20:41:01,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ebdaf94e46cf4648b76714405b522625, entries=150, sequenceid=518, filesize=12.0 K 2024-11-11T20:41:01,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 22891ca6e2fbd305fc3d0297f1195419 in 1263ms, sequenceid=518, compaction requested=true 2024-11-11T20:41:01,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:41:01,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:01,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:01,776 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:01,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:01,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:01,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22891ca6e2fbd305fc3d0297f1195419:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:01,777 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:01,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:01,778 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:01,778 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:01,778 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/A is initiating minor compaction (all files) 2024-11-11T20:41:01,778 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/B is initiating minor compaction (all files) 2024-11-11T20:41:01,778 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/B in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,778 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/A in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,778 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ec73ee6b0d584d4380b7e86a8244eb9a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/b92949e66b3a4837a53a0d218e9d82c1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1572d47d1c604aae8fc10bc7f419bef5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/dd5e1655f2a54b38acc1e123bad16e61] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=58.6 K 2024-11-11T20:41:01,778 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/002caa58323945368bd01e8f4984e756, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/d87d5811313547b485e3d7ac2078ca0c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/8bd67c0b6ed745a091a1445063ca4c9e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3ee7a872b55b4c0793fac7e02f034a87] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=49.1 K 2024-11-11T20:41:01,778 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec73ee6b0d584d4380b7e86a8244eb9a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731357657396 2024-11-11T20:41:01,778 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 002caa58323945368bd01e8f4984e756, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731357657396 2024-11-11T20:41:01,779 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b92949e66b3a4837a53a0d218e9d82c1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731357658071 2024-11-11T20:41:01,779 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d87d5811313547b485e3d7ac2078ca0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731357658071 2024-11-11T20:41:01,779 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1572d47d1c604aae8fc10bc7f419bef5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731357658717 2024-11-11T20:41:01,779 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bd67c0b6ed745a091a1445063ca4c9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731357658717 2024-11-11T20:41:01,779 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ee7a872b55b4c0793fac7e02f034a87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731357659372 2024-11-11T20:41:01,779 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd5e1655f2a54b38acc1e123bad16e61, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731357659371 2024-11-11T20:41:01,792 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#B#compaction#324 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:01,793 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/f9719a31ef2f49b6b26c706b562eeba7 is 50, key is test_row_0/B:col10/1731357659373/Put/seqid=0 2024-11-11T20:41:01,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742201_1377 (size=13527) 2024-11-11T20:41:01,801 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#A#compaction#325 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:01,801 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/ee1a847f8efa47f4ad4340dc415e1da5 is 50, key is test_row_0/A:col10/1731357659373/Put/seqid=0 2024-11-11T20:41:01,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742202_1378 (size=13527) 2024-11-11T20:41:01,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-11T20:41:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:01,910 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:41:01,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/310301bb2e1d42f6ba78c70499896e4d is 50, key is test_row_0/A:col10/1731357660520/Put/seqid=0 2024-11-11T20:41:01,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742203_1379 (size=12301) 2024-11-11T20:41:01,920 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/310301bb2e1d42f6ba78c70499896e4d 2024-11-11T20:41:01,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/ac3bd9b471324bf9bd53ed39a6f213a6 is 50, key is test_row_0/B:col10/1731357660520/Put/seqid=0 2024-11-11T20:41:01,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742204_1380 (size=12301) 2024-11-11T20:41:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T20:41:02,206 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/f9719a31ef2f49b6b26c706b562eeba7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/f9719a31ef2f49b6b26c706b562eeba7 2024-11-11T20:41:02,211 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/ee1a847f8efa47f4ad4340dc415e1da5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ee1a847f8efa47f4ad4340dc415e1da5 2024-11-11T20:41:02,212 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/B of 22891ca6e2fbd305fc3d0297f1195419 into f9719a31ef2f49b6b26c706b562eeba7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:02,212 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:41:02,212 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/B, priority=12, startTime=1731357661776; duration=0sec 2024-11-11T20:41:02,212 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:02,212 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:B 2024-11-11T20:41:02,212 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:02,213 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:02,213 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 22891ca6e2fbd305fc3d0297f1195419/C is initiating minor compaction (all files) 2024-11-11T20:41:02,213 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22891ca6e2fbd305fc3d0297f1195419/C in TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:02,213 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c52dd230f96a4579bf81f59ba84cabd3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/46c47b35944946b786ca804c3b7c7506, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/e5d373ba3ec248c9b08ddd84ddcf41cf, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ebdaf94e46cf4648b76714405b522625] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp, totalSize=49.1 K 2024-11-11T20:41:02,214 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c52dd230f96a4579bf81f59ba84cabd3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731357657396 2024-11-11T20:41:02,214 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 46c47b35944946b786ca804c3b7c7506, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731357658071 2024-11-11T20:41:02,214 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d373ba3ec248c9b08ddd84ddcf41cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731357658717 2024-11-11T20:41:02,214 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ebdaf94e46cf4648b76714405b522625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731357659372 2024-11-11T20:41:02,215 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/A of 22891ca6e2fbd305fc3d0297f1195419 into ee1a847f8efa47f4ad4340dc415e1da5(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:02,215 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:41:02,215 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/A, priority=12, startTime=1731357661776; duration=0sec 2024-11-11T20:41:02,215 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:02,215 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:A 2024-11-11T20:41:02,221 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22891ca6e2fbd305fc3d0297f1195419#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:02,221 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/17cfc159fc7b46c6ad132caab2910a30 is 50, key is test_row_0/C:col10/1731357659373/Put/seqid=0 2024-11-11T20:41:02,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742205_1381 (size=13527) 2024-11-11T20:41:02,335 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/ac3bd9b471324bf9bd53ed39a6f213a6 2024-11-11T20:41:02,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/dbacca4ca33847d89231342e839c84f1 is 50, key is test_row_0/C:col10/1731357660520/Put/seqid=0 2024-11-11T20:41:02,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742206_1382 (size=12301) 2024-11-11T20:41:02,637 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/17cfc159fc7b46c6ad132caab2910a30 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/17cfc159fc7b46c6ad132caab2910a30 2024-11-11T20:41:02,645 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22891ca6e2fbd305fc3d0297f1195419/C of 22891ca6e2fbd305fc3d0297f1195419 into 17cfc159fc7b46c6ad132caab2910a30(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:02,645 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:41:02,645 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., storeName=22891ca6e2fbd305fc3d0297f1195419/C, priority=12, startTime=1731357661776; duration=0sec 2024-11-11T20:41:02,645 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:02,645 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22891ca6e2fbd305fc3d0297f1195419:C 2024-11-11T20:41:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:41:02,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. as already flushing 2024-11-11T20:41:02,672 INFO [Thread-1235 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1235. 2024-11-11T20:41:02,672 INFO [Thread-1239 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1239. 2024-11-11T20:41:02,672 INFO [Thread-1233 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1233. 2024-11-11T20:41:02,672 DEBUG [Thread-1239 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:02,672 DEBUG [Thread-1235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:02,672 DEBUG [Thread-1233 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:02,672 DEBUG [Thread-1239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,672 DEBUG [Thread-1235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,672 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,672 DEBUG [Thread-1239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,672 DEBUG [Thread-1235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,672 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:02,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:02,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:02,673 INFO [Thread-1231 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1231. 2024-11-11T20:41:02,673 DEBUG [Thread-1231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:02,674 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,674 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,674 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:02,681 INFO [Thread-1237 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1237. 2024-11-11T20:41:02,681 DEBUG [Thread-1237 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:02,681 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,681 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:02,748 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/dbacca4ca33847d89231342e839c84f1 2024-11-11T20:41:02,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/310301bb2e1d42f6ba78c70499896e4d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/310301bb2e1d42f6ba78c70499896e4d 2024-11-11T20:41:02,760 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/310301bb2e1d42f6ba78c70499896e4d, entries=150, sequenceid=529, filesize=12.0 K 2024-11-11T20:41:02,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/ac3bd9b471324bf9bd53ed39a6f213a6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ac3bd9b471324bf9bd53ed39a6f213a6 2024-11-11T20:41:02,766 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ac3bd9b471324bf9bd53ed39a6f213a6, entries=150, sequenceid=529, filesize=12.0 K 2024-11-11T20:41:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/dbacca4ca33847d89231342e839c84f1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/dbacca4ca33847d89231342e839c84f1 2024-11-11T20:41:02,771 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/dbacca4ca33847d89231342e839c84f1, entries=150, sequenceid=529, filesize=12.0 K 2024-11-11T20:41:02,772 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=33.54 KB/34350 for 22891ca6e2fbd305fc3d0297f1195419 in 863ms, sequenceid=529, compaction requested=false 2024-11-11T20:41:02,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2603): Flush status journal for 22891ca6e2fbd305fc3d0297f1195419: 2024-11-11T20:41:02,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:02,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=93 2024-11-11T20:41:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=93 2024-11-11T20:41:02,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-11T20:41:02,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9390 sec 2024-11-11T20:41:02,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees in 1.9430 sec 2024-11-11T20:41:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T20:41:02,982 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3508 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3491 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3478 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3516 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3484 2024-11-11T20:41:02,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:41:02,983 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:41:02,984 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:41:02,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:02,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:02,986 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:41:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-11T20:41:02,991 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357662991"}]},"ts":"1731357662991"} 2024-11-11T20:41:02,992 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:41:02,992 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:41:02,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:41:02,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22891ca6e2fbd305fc3d0297f1195419, UNASSIGN}] 2024-11-11T20:41:02,996 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22891ca6e2fbd305fc3d0297f1195419, UNASSIGN 2024-11-11T20:41:02,996 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=22891ca6e2fbd305fc3d0297f1195419, regionState=CLOSING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:02,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22891ca6e2fbd305fc3d0297f1195419, UNASSIGN because future has completed 2024-11-11T20:41:02,998 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:41:02,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:41:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-11T20:41:03,151 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:41:03,152 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:41:03,152 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 22891ca6e2fbd305fc3d0297f1195419, disabling compactions & flushes 2024-11-11T20:41:03,152 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:03,152 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:03,152 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. after waiting 0 ms 2024-11-11T20:41:03,152 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:03,152 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(2902): Flushing 22891ca6e2fbd305fc3d0297f1195419 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T20:41:03,156 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/71dd13d58a73475c8f3fa302a4feeeaf is 50, key is test_row_0/A:col10/1731357662671/Put/seqid=0 2024-11-11T20:41:03,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742207_1383 (size=9857) 2024-11-11T20:41:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-11T20:41:03,560 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/71dd13d58a73475c8f3fa302a4feeeaf 2024-11-11T20:41:03,571 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/4553d1a0c32245e785c541fb6ee54049 is 50, key is test_row_0/B:col10/1731357662671/Put/seqid=0 2024-11-11T20:41:03,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742208_1384 (size=9857) 2024-11-11T20:41:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-11T20:41:03,976 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/4553d1a0c32245e785c541fb6ee54049 2024-11-11T20:41:03,992 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/4ec820b85b994ea5a48d4d715daac2ce is 50, key is test_row_0/C:col10/1731357662671/Put/seqid=0 2024-11-11T20:41:03,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742209_1385 (size=9857) 2024-11-11T20:41:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-11T20:41:04,397 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/4ec820b85b994ea5a48d4d715daac2ce 2024-11-11T20:41:04,407 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/A/71dd13d58a73475c8f3fa302a4feeeaf as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/71dd13d58a73475c8f3fa302a4feeeaf 2024-11-11T20:41:04,414 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/71dd13d58a73475c8f3fa302a4feeeaf, entries=100, sequenceid=540, filesize=9.6 K 2024-11-11T20:41:04,416 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/B/4553d1a0c32245e785c541fb6ee54049 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4553d1a0c32245e785c541fb6ee54049 2024-11-11T20:41:04,420 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4553d1a0c32245e785c541fb6ee54049, entries=100, sequenceid=540, filesize=9.6 K 2024-11-11T20:41:04,421 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/.tmp/C/4ec820b85b994ea5a48d4d715daac2ce as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/4ec820b85b994ea5a48d4d715daac2ce 2024-11-11T20:41:04,424 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/4ec820b85b994ea5a48d4d715daac2ce, entries=100, sequenceid=540, filesize=9.6 K 2024-11-11T20:41:04,425 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 22891ca6e2fbd305fc3d0297f1195419 in 1273ms, sequenceid=540, compaction requested=true 2024-11-11T20:41:04,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/11538f220f5042df83afbdde1e9f6f97, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d934c1304bdb41c8a92f182bb9e6f3ad, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e327574638184729972e253b66c161b2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/3aa49e5c521e43c696f69342fe209153, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/888f240c7f774e129376c748c2910e5b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/50a69978fc254afe9d9e698bf4feda5f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e0e3f61a3ad84a3685f1035dc1f4ba56, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/8d3e2e9644644ed384ffc685160d7864, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1b07b29f5cd542d3aa99013556bebeb0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0c48a08a4ec14962b7f773ce681eb4f1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0ccba9d26e6d4722980eaa8f6a9a3d9e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/85db523c4e904a98b7e6fc44a3c86766, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2e18a1382b224fef9d3b1c2bdc9d306b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/58a2397bbcd94c79a445acf2b6544d11, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7a3ffc94ebdf4b95830c76c8c4b473ca, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7fc9d788db9a44999be6aae622ba1e7c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/a6a53caab4bd43fd848e0a31dec8a2eb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6c15e6b440a64b6c83104096caf845e9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0a2d81281fe54ae18a7dd4d58ccc5e36, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ffae07ead4dc4c55b3f77c06201c6646, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/84214aa3a14d418baba4357b3f8ace10, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d226242d19ab47808999d46a3057e418, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6ff2b89380364b5692eae446dc33c8ff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0742949245f44eefb799a3bb52fb6d0b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0eaa5e653d004a3e8680132b2dd38208, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/39185fa0b56b4c0fbf5e11e55beabb80, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ed00517c999c487ab79708b331f9f333, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/fbab4fe58c6a4b1fa0b25df7f849032c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1ee46546bf65422798af58e4900647f5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2b3e167618bd4686b94c3346ab9425c2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d9e70ac682964a9eb4bec65067253273, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ec73ee6b0d584d4380b7e86a8244eb9a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/4142b69cafa340efa9906d291d76d6c5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/b92949e66b3a4837a53a0d218e9d82c1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1572d47d1c604aae8fc10bc7f419bef5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/dd5e1655f2a54b38acc1e123bad16e61] to archive 2024-11-11T20:41:04,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:41:04,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/11538f220f5042df83afbdde1e9f6f97 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/11538f220f5042df83afbdde1e9f6f97 2024-11-11T20:41:04,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d934c1304bdb41c8a92f182bb9e6f3ad to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d934c1304bdb41c8a92f182bb9e6f3ad 2024-11-11T20:41:04,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e327574638184729972e253b66c161b2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e327574638184729972e253b66c161b2 2024-11-11T20:41:04,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/3aa49e5c521e43c696f69342fe209153 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/3aa49e5c521e43c696f69342fe209153 2024-11-11T20:41:04,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/888f240c7f774e129376c748c2910e5b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/888f240c7f774e129376c748c2910e5b 2024-11-11T20:41:04,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/50a69978fc254afe9d9e698bf4feda5f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/50a69978fc254afe9d9e698bf4feda5f 2024-11-11T20:41:04,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e0e3f61a3ad84a3685f1035dc1f4ba56 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/e0e3f61a3ad84a3685f1035dc1f4ba56 2024-11-11T20:41:04,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/8d3e2e9644644ed384ffc685160d7864 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/8d3e2e9644644ed384ffc685160d7864 2024-11-11T20:41:04,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1b07b29f5cd542d3aa99013556bebeb0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1b07b29f5cd542d3aa99013556bebeb0 2024-11-11T20:41:04,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0c48a08a4ec14962b7f773ce681eb4f1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0c48a08a4ec14962b7f773ce681eb4f1 2024-11-11T20:41:04,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0ccba9d26e6d4722980eaa8f6a9a3d9e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0ccba9d26e6d4722980eaa8f6a9a3d9e 2024-11-11T20:41:04,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/85db523c4e904a98b7e6fc44a3c86766 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/85db523c4e904a98b7e6fc44a3c86766 2024-11-11T20:41:04,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2e18a1382b224fef9d3b1c2bdc9d306b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2e18a1382b224fef9d3b1c2bdc9d306b 2024-11-11T20:41:04,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/58a2397bbcd94c79a445acf2b6544d11 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/58a2397bbcd94c79a445acf2b6544d11 2024-11-11T20:41:04,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7a3ffc94ebdf4b95830c76c8c4b473ca to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7a3ffc94ebdf4b95830c76c8c4b473ca 2024-11-11T20:41:04,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7fc9d788db9a44999be6aae622ba1e7c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/7fc9d788db9a44999be6aae622ba1e7c 2024-11-11T20:41:04,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/a6a53caab4bd43fd848e0a31dec8a2eb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/a6a53caab4bd43fd848e0a31dec8a2eb 2024-11-11T20:41:04,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6c15e6b440a64b6c83104096caf845e9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6c15e6b440a64b6c83104096caf845e9 2024-11-11T20:41:04,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0a2d81281fe54ae18a7dd4d58ccc5e36 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0a2d81281fe54ae18a7dd4d58ccc5e36 2024-11-11T20:41:04,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ffae07ead4dc4c55b3f77c06201c6646 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ffae07ead4dc4c55b3f77c06201c6646 2024-11-11T20:41:04,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/84214aa3a14d418baba4357b3f8ace10 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/84214aa3a14d418baba4357b3f8ace10 2024-11-11T20:41:04,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d226242d19ab47808999d46a3057e418 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d226242d19ab47808999d46a3057e418 2024-11-11T20:41:04,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6ff2b89380364b5692eae446dc33c8ff to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/6ff2b89380364b5692eae446dc33c8ff 2024-11-11T20:41:04,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0742949245f44eefb799a3bb52fb6d0b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0742949245f44eefb799a3bb52fb6d0b 2024-11-11T20:41:04,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0eaa5e653d004a3e8680132b2dd38208 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/0eaa5e653d004a3e8680132b2dd38208 2024-11-11T20:41:04,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/39185fa0b56b4c0fbf5e11e55beabb80 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/39185fa0b56b4c0fbf5e11e55beabb80 2024-11-11T20:41:04,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ed00517c999c487ab79708b331f9f333 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ed00517c999c487ab79708b331f9f333 2024-11-11T20:41:04,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/fbab4fe58c6a4b1fa0b25df7f849032c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/fbab4fe58c6a4b1fa0b25df7f849032c 2024-11-11T20:41:04,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1ee46546bf65422798af58e4900647f5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1ee46546bf65422798af58e4900647f5 2024-11-11T20:41:04,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2b3e167618bd4686b94c3346ab9425c2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/2b3e167618bd4686b94c3346ab9425c2 2024-11-11T20:41:04,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d9e70ac682964a9eb4bec65067253273 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/d9e70ac682964a9eb4bec65067253273 2024-11-11T20:41:04,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ec73ee6b0d584d4380b7e86a8244eb9a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ec73ee6b0d584d4380b7e86a8244eb9a 2024-11-11T20:41:04,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/4142b69cafa340efa9906d291d76d6c5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/4142b69cafa340efa9906d291d76d6c5 2024-11-11T20:41:04,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/b92949e66b3a4837a53a0d218e9d82c1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/b92949e66b3a4837a53a0d218e9d82c1 2024-11-11T20:41:04,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1572d47d1c604aae8fc10bc7f419bef5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/1572d47d1c604aae8fc10bc7f419bef5 2024-11-11T20:41:04,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/dd5e1655f2a54b38acc1e123bad16e61 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/dd5e1655f2a54b38acc1e123bad16e61 2024-11-11T20:41:04,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/53d1e9c512a84e548a2ffbc2de232286, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fda69e0588fa4d989bcc91aba2184b95, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/aeabc5c8314d47a5ab094db323940bc6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5ad7a7a79d2d459891a0fa6202d4de63, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4b1099936d5842339630a0b6bb5ce0b5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dcb720b25ada4912b81a786af3332d0a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ab428dcb4daf4ed2af71c71d25ed79bd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/db02c4ee66b447a099939b0488edb5dc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/76c3314dd4ca49718060c135ea820fca, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ea5351008ad1424e9abbf5d058dff66b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/2a124d31c6374a708cf4af494f3b1c62, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5e88fc7bbcbc438481a62f473e9e312f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/50ef00f865b5460eac0887b7f9b33e38, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fc592c98679843c48a211f6e7179cac1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/eebd89a050244cd1a8a20c4d123f4d10, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0e00260cba5842548d7e35eb0faf5b94, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/948241d031cc40f5b7ce5c538e595cd3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3660274e6b6b4bdebbb1eedc3bbe2cbb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/b9eaeda299d64cdd8ad8252fce1e3cdb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dd19b5290a68409c86be2dba522664c5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/252870b393b04af29b49a80cdfc522fc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/442958689209493898e830949f92ee3a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/17687023fd0a4bf98abf0b78cb76efa4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6cbfb1ff45dc49f0b223425ad88e1c80, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/f83ec3941e854733883253da3a2d3343, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/679202a7eef74e6ea14e1d63a877a791, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/66902a1076574e8eb68c818376f59dd6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/9ca244cc70654cfbb6f94e5bbe5cf5b4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6d040f96cb6e4d6a8175746ee468bb8a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/013f3ff86a7442ee809b5570fcbc424e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6c418b0362624714963f513332b4fd88, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/002caa58323945368bd01e8f4984e756, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0d5801718f3b4f7cb36709051514554b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/d87d5811313547b485e3d7ac2078ca0c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/8bd67c0b6ed745a091a1445063ca4c9e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3ee7a872b55b4c0793fac7e02f034a87] to archive 2024-11-11T20:41:04,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:41:04,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/53d1e9c512a84e548a2ffbc2de232286 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/53d1e9c512a84e548a2ffbc2de232286 2024-11-11T20:41:04,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fda69e0588fa4d989bcc91aba2184b95 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fda69e0588fa4d989bcc91aba2184b95 2024-11-11T20:41:04,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/aeabc5c8314d47a5ab094db323940bc6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/aeabc5c8314d47a5ab094db323940bc6 2024-11-11T20:41:04,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5ad7a7a79d2d459891a0fa6202d4de63 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5ad7a7a79d2d459891a0fa6202d4de63 2024-11-11T20:41:04,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4b1099936d5842339630a0b6bb5ce0b5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4b1099936d5842339630a0b6bb5ce0b5 2024-11-11T20:41:04,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dcb720b25ada4912b81a786af3332d0a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dcb720b25ada4912b81a786af3332d0a 2024-11-11T20:41:04,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ab428dcb4daf4ed2af71c71d25ed79bd to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ab428dcb4daf4ed2af71c71d25ed79bd 2024-11-11T20:41:04,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/db02c4ee66b447a099939b0488edb5dc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/db02c4ee66b447a099939b0488edb5dc 2024-11-11T20:41:04,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/76c3314dd4ca49718060c135ea820fca to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/76c3314dd4ca49718060c135ea820fca 2024-11-11T20:41:04,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ea5351008ad1424e9abbf5d058dff66b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ea5351008ad1424e9abbf5d058dff66b 2024-11-11T20:41:04,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/2a124d31c6374a708cf4af494f3b1c62 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/2a124d31c6374a708cf4af494f3b1c62 2024-11-11T20:41:04,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5e88fc7bbcbc438481a62f473e9e312f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/5e88fc7bbcbc438481a62f473e9e312f 2024-11-11T20:41:04,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/50ef00f865b5460eac0887b7f9b33e38 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/50ef00f865b5460eac0887b7f9b33e38 2024-11-11T20:41:04,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fc592c98679843c48a211f6e7179cac1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/fc592c98679843c48a211f6e7179cac1 2024-11-11T20:41:04,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/eebd89a050244cd1a8a20c4d123f4d10 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/eebd89a050244cd1a8a20c4d123f4d10 2024-11-11T20:41:04,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0e00260cba5842548d7e35eb0faf5b94 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0e00260cba5842548d7e35eb0faf5b94 2024-11-11T20:41:04,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/948241d031cc40f5b7ce5c538e595cd3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/948241d031cc40f5b7ce5c538e595cd3 2024-11-11T20:41:04,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3660274e6b6b4bdebbb1eedc3bbe2cbb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3660274e6b6b4bdebbb1eedc3bbe2cbb 2024-11-11T20:41:04,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/b9eaeda299d64cdd8ad8252fce1e3cdb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/b9eaeda299d64cdd8ad8252fce1e3cdb 2024-11-11T20:41:04,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dd19b5290a68409c86be2dba522664c5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/dd19b5290a68409c86be2dba522664c5 2024-11-11T20:41:04,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/252870b393b04af29b49a80cdfc522fc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/252870b393b04af29b49a80cdfc522fc 2024-11-11T20:41:04,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/442958689209493898e830949f92ee3a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/442958689209493898e830949f92ee3a 2024-11-11T20:41:04,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/17687023fd0a4bf98abf0b78cb76efa4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/17687023fd0a4bf98abf0b78cb76efa4 2024-11-11T20:41:04,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6cbfb1ff45dc49f0b223425ad88e1c80 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6cbfb1ff45dc49f0b223425ad88e1c80 2024-11-11T20:41:04,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/f83ec3941e854733883253da3a2d3343 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/f83ec3941e854733883253da3a2d3343 2024-11-11T20:41:04,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/679202a7eef74e6ea14e1d63a877a791 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/679202a7eef74e6ea14e1d63a877a791 2024-11-11T20:41:04,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/66902a1076574e8eb68c818376f59dd6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/66902a1076574e8eb68c818376f59dd6 2024-11-11T20:41:04,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/9ca244cc70654cfbb6f94e5bbe5cf5b4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/9ca244cc70654cfbb6f94e5bbe5cf5b4 2024-11-11T20:41:04,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6d040f96cb6e4d6a8175746ee468bb8a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6d040f96cb6e4d6a8175746ee468bb8a 2024-11-11T20:41:04,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/013f3ff86a7442ee809b5570fcbc424e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/013f3ff86a7442ee809b5570fcbc424e 2024-11-11T20:41:04,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6c418b0362624714963f513332b4fd88 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/6c418b0362624714963f513332b4fd88 2024-11-11T20:41:04,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/002caa58323945368bd01e8f4984e756 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/002caa58323945368bd01e8f4984e756 2024-11-11T20:41:04,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0d5801718f3b4f7cb36709051514554b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/0d5801718f3b4f7cb36709051514554b 2024-11-11T20:41:04,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/d87d5811313547b485e3d7ac2078ca0c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/d87d5811313547b485e3d7ac2078ca0c 2024-11-11T20:41:04,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/8bd67c0b6ed745a091a1445063ca4c9e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/8bd67c0b6ed745a091a1445063ca4c9e 2024-11-11T20:41:04,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3ee7a872b55b4c0793fac7e02f034a87 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/3ee7a872b55b4c0793fac7e02f034a87 2024-11-11T20:41:04,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/1719ae5c7fe04b2d8c42794db709ef50, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ab0240621cf447b781cf2c3ccc2c12b0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d1c01da880f749aab2c188ffaf3612a9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d15ad5ae63e84c5ca634acf24971bb20, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ade51343f1af436e8e55774a8f16d13b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d56312a3fc7545c29c7d4e51a3d79816, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/369d4a4c61d54f54a9677f845cb1e9a0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c189e703520c454d8e2fca960f438be1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b4005d7970324558b1ffa7c1b7606fdc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/910eb3e706fd4ebcbdc35141f82c551d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ea87db2925af483aa4e9b72daf90876c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/82ee3d24f7384350a69a86ec2ab5bc38, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8f203dd4387544208a8b27d42568f480, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8949a14ba1f949358d94f1a022149fb6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fa8d0ba375f04e158456214ba7e057c3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/6f87fdcdfa44430ab82ccc99b9ee1664, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b61b7b9477274af39f56d33f145ed7e5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fc7a699671a1455d8a5b6e3f96e043f6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a1b5987f2adf4f2fb8b308918dd4e144, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8ea295098e1346b7b92a289dd85a03c6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b9172f42fd76432f88217a3896eb79ef, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/85527e8b024545e8a6eda5a3f3286239, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/12e94d9a0e254dc39ff1cd83307153cb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/3ab0d55567aa4edbb815e5975214009b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a89d1f922c4b423692cff3afb1513710, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/418b2654bb834f3d8234cd11296fe2a3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bf30752562a141aebd08e5b59d71cb20, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/5b86c37642fe4dedbe627be4264daaef, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b7b3f49ee5b14bf3a63b8736d55db50a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b69046df8ceb49cdb22f90a849c4f59a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8902b7d41c18413997c5f3c330569b63, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c52dd230f96a4579bf81f59ba84cabd3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bdf689268a0d4a6c882695f42ad870da, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/46c47b35944946b786ca804c3b7c7506, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/e5d373ba3ec248c9b08ddd84ddcf41cf, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ebdaf94e46cf4648b76714405b522625] to archive 2024-11-11T20:41:04,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:41:04,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/1719ae5c7fe04b2d8c42794db709ef50 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/1719ae5c7fe04b2d8c42794db709ef50 2024-11-11T20:41:04,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ab0240621cf447b781cf2c3ccc2c12b0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ab0240621cf447b781cf2c3ccc2c12b0 2024-11-11T20:41:04,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d1c01da880f749aab2c188ffaf3612a9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d1c01da880f749aab2c188ffaf3612a9 2024-11-11T20:41:04,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d15ad5ae63e84c5ca634acf24971bb20 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d15ad5ae63e84c5ca634acf24971bb20 2024-11-11T20:41:04,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ade51343f1af436e8e55774a8f16d13b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ade51343f1af436e8e55774a8f16d13b 2024-11-11T20:41:04,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d56312a3fc7545c29c7d4e51a3d79816 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/d56312a3fc7545c29c7d4e51a3d79816 2024-11-11T20:41:04,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/369d4a4c61d54f54a9677f845cb1e9a0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/369d4a4c61d54f54a9677f845cb1e9a0 2024-11-11T20:41:04,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c189e703520c454d8e2fca960f438be1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c189e703520c454d8e2fca960f438be1 2024-11-11T20:41:04,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b4005d7970324558b1ffa7c1b7606fdc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b4005d7970324558b1ffa7c1b7606fdc 2024-11-11T20:41:04,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/910eb3e706fd4ebcbdc35141f82c551d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/910eb3e706fd4ebcbdc35141f82c551d 2024-11-11T20:41:04,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ea87db2925af483aa4e9b72daf90876c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ea87db2925af483aa4e9b72daf90876c 2024-11-11T20:41:04,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/82ee3d24f7384350a69a86ec2ab5bc38 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/82ee3d24f7384350a69a86ec2ab5bc38 2024-11-11T20:41:04,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8f203dd4387544208a8b27d42568f480 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8f203dd4387544208a8b27d42568f480 2024-11-11T20:41:04,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8949a14ba1f949358d94f1a022149fb6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8949a14ba1f949358d94f1a022149fb6 2024-11-11T20:41:04,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fa8d0ba375f04e158456214ba7e057c3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fa8d0ba375f04e158456214ba7e057c3 2024-11-11T20:41:04,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/6f87fdcdfa44430ab82ccc99b9ee1664 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/6f87fdcdfa44430ab82ccc99b9ee1664 2024-11-11T20:41:04,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b61b7b9477274af39f56d33f145ed7e5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b61b7b9477274af39f56d33f145ed7e5 2024-11-11T20:41:04,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fc7a699671a1455d8a5b6e3f96e043f6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/fc7a699671a1455d8a5b6e3f96e043f6 2024-11-11T20:41:04,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a1b5987f2adf4f2fb8b308918dd4e144 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a1b5987f2adf4f2fb8b308918dd4e144 2024-11-11T20:41:04,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8ea295098e1346b7b92a289dd85a03c6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8ea295098e1346b7b92a289dd85a03c6 2024-11-11T20:41:04,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b9172f42fd76432f88217a3896eb79ef to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b9172f42fd76432f88217a3896eb79ef 2024-11-11T20:41:04,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/85527e8b024545e8a6eda5a3f3286239 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/85527e8b024545e8a6eda5a3f3286239 2024-11-11T20:41:04,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/12e94d9a0e254dc39ff1cd83307153cb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/12e94d9a0e254dc39ff1cd83307153cb 2024-11-11T20:41:04,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/3ab0d55567aa4edbb815e5975214009b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/3ab0d55567aa4edbb815e5975214009b 2024-11-11T20:41:04,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a89d1f922c4b423692cff3afb1513710 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/a89d1f922c4b423692cff3afb1513710 2024-11-11T20:41:04,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/418b2654bb834f3d8234cd11296fe2a3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/418b2654bb834f3d8234cd11296fe2a3 2024-11-11T20:41:04,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bf30752562a141aebd08e5b59d71cb20 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bf30752562a141aebd08e5b59d71cb20 2024-11-11T20:41:04,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/5b86c37642fe4dedbe627be4264daaef to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/5b86c37642fe4dedbe627be4264daaef 2024-11-11T20:41:04,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b7b3f49ee5b14bf3a63b8736d55db50a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b7b3f49ee5b14bf3a63b8736d55db50a 2024-11-11T20:41:04,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b69046df8ceb49cdb22f90a849c4f59a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/b69046df8ceb49cdb22f90a849c4f59a 2024-11-11T20:41:04,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8902b7d41c18413997c5f3c330569b63 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/8902b7d41c18413997c5f3c330569b63 2024-11-11T20:41:04,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c52dd230f96a4579bf81f59ba84cabd3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/c52dd230f96a4579bf81f59ba84cabd3 2024-11-11T20:41:04,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bdf689268a0d4a6c882695f42ad870da to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/bdf689268a0d4a6c882695f42ad870da 2024-11-11T20:41:04,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/46c47b35944946b786ca804c3b7c7506 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/46c47b35944946b786ca804c3b7c7506 2024-11-11T20:41:04,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/e5d373ba3ec248c9b08ddd84ddcf41cf to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/e5d373ba3ec248c9b08ddd84ddcf41cf 2024-11-11T20:41:04,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ebdaf94e46cf4648b76714405b522625 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/ebdaf94e46cf4648b76714405b522625 2024-11-11T20:41:04,517 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/recovered.edits/543.seqid, newMaxSeqId=543, maxSeqId=1 2024-11-11T20:41:04,518 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. 2024-11-11T20:41:04,518 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 22891ca6e2fbd305fc3d0297f1195419: Waiting for close lock at 1731357663152Running coprocessor pre-close hooks at 1731357663152Disabling compacts and flushes for region at 1731357663152Disabling writes for close at 1731357663152Obtaining lock to block concurrent updates at 1731357663152Preparing flush snapshotting stores in 22891ca6e2fbd305fc3d0297f1195419 at 1731357663152Finished memstore snapshotting TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731357663153 (+1 ms)Flushing stores of TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419. at 1731357663153Flushing 22891ca6e2fbd305fc3d0297f1195419/A: creating writer at 1731357663153Flushing 22891ca6e2fbd305fc3d0297f1195419/A: appending metadata at 1731357663156 (+3 ms)Flushing 22891ca6e2fbd305fc3d0297f1195419/A: closing flushed file at 1731357663156Flushing 22891ca6e2fbd305fc3d0297f1195419/B: creating writer at 1731357663567 (+411 ms)Flushing 22891ca6e2fbd305fc3d0297f1195419/B: appending metadata at 1731357663570 (+3 ms)Flushing 22891ca6e2fbd305fc3d0297f1195419/B: closing flushed file at 1731357663570Flushing 22891ca6e2fbd305fc3d0297f1195419/C: creating writer at 1731357663985 (+415 ms)Flushing 22891ca6e2fbd305fc3d0297f1195419/C: appending metadata at 1731357663990 (+5 ms)Flushing 22891ca6e2fbd305fc3d0297f1195419/C: closing flushed file at 1731357663991 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@749e0a0b: reopening flushed file at 1731357664406 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ab1cfd4: reopening flushed file at 1731357664415 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@805b3bc: reopening flushed file at 1731357664420 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 22891ca6e2fbd305fc3d0297f1195419 in 1273ms, sequenceid=540, compaction requested=true at 1731357664425 (+5 ms)Writing region close event to WAL at 1731357664514 (+89 ms)Running coprocessor post-close hooks at 1731357664518 (+4 ms)Closed at 1731357664518 2024-11-11T20:41:04,520 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:41:04,520 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=22891ca6e2fbd305fc3d0297f1195419, regionState=CLOSED 2024-11-11T20:41:04,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:41:04,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-11T20:41:04,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 22891ca6e2fbd305fc3d0297f1195419, server=51ca66f7ee3c,44271,1731357576703 in 1.5250 sec 2024-11-11T20:41:04,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T20:41:04,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22891ca6e2fbd305fc3d0297f1195419, UNASSIGN in 1.5300 sec 2024-11-11T20:41:04,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-11T20:41:04,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5350 sec 2024-11-11T20:41:04,531 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357664531"}]},"ts":"1731357664531"} 2024-11-11T20:41:04,533 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:41:04,533 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:41:04,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5470 sec 2024-11-11T20:41:05,052 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:41:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-11T20:41:05,142 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:05,143 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:41:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:05,148 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-11T20:41:05,150 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=98, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:05,154 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:41:05,156 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/recovered.edits] 2024-11-11T20:41:05,157 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/310301bb2e1d42f6ba78c70499896e4d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/310301bb2e1d42f6ba78c70499896e4d 2024-11-11T20:41:05,158 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/71dd13d58a73475c8f3fa302a4feeeaf to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/71dd13d58a73475c8f3fa302a4feeeaf 2024-11-11T20:41:05,159 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ee1a847f8efa47f4ad4340dc415e1da5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/A/ee1a847f8efa47f4ad4340dc415e1da5 2024-11-11T20:41:05,161 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4553d1a0c32245e785c541fb6ee54049 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/4553d1a0c32245e785c541fb6ee54049 2024-11-11T20:41:05,162 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ac3bd9b471324bf9bd53ed39a6f213a6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/ac3bd9b471324bf9bd53ed39a6f213a6 2024-11-11T20:41:05,163 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/f9719a31ef2f49b6b26c706b562eeba7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/B/f9719a31ef2f49b6b26c706b562eeba7 2024-11-11T20:41:05,164 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/17cfc159fc7b46c6ad132caab2910a30 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/17cfc159fc7b46c6ad132caab2910a30 2024-11-11T20:41:05,165 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/4ec820b85b994ea5a48d4d715daac2ce to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/4ec820b85b994ea5a48d4d715daac2ce 2024-11-11T20:41:05,166 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/dbacca4ca33847d89231342e839c84f1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/C/dbacca4ca33847d89231342e839c84f1 2024-11-11T20:41:05,169 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/recovered.edits/543.seqid to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419/recovered.edits/543.seqid 2024-11-11T20:41:05,170 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/22891ca6e2fbd305fc3d0297f1195419 2024-11-11T20:41:05,170 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:41:05,173 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=98, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:05,177 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:41:05,179 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:41:05,181 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=98, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:05,181 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:41:05,181 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731357665181"}]},"ts":"9223372036854775807"} 2024-11-11T20:41:05,183 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:41:05,183 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 22891ca6e2fbd305fc3d0297f1195419, NAME => 'TestAcidGuarantees,,1731357639490.22891ca6e2fbd305fc3d0297f1195419.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:41:05,183 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:41:05,183 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731357665183"}]},"ts":"9223372036854775807"} 2024-11-11T20:41:05,185 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:41:05,185 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=98, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:05,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-11T20:41:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-11T20:41:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:41:05,262 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:05,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 242), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=444 (was 372) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4517 (was 4573) 2024-11-11T20:41:05,284 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=444, ProcessCount=11, AvailableMemoryMB=4516 2024-11-11T20:41:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:41:05,286 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:41:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:05,288 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:41:05,288 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:05,288 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 99 2024-11-11T20:41:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:41:05,289 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:41:05,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742210_1386 (size=959) 2024-11-11T20:41:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:41:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:41:05,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30 2024-11-11T20:41:05,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742211_1387 (size=53) 2024-11-11T20:41:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:41:06,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:41:06,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fe28f5753443ca2f28d81a625707f258, disabling compactions & flushes 2024-11-11T20:41:06,108 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:06,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:06,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. after waiting 0 ms 2024-11-11T20:41:06,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:06,108 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:06,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe28f5753443ca2f28d81a625707f258: Waiting for close lock at 1731357666108Disabling compacts and flushes for region at 1731357666108Disabling writes for close at 1731357666108Writing region close event to WAL at 1731357666108Closed at 1731357666108 2024-11-11T20:41:06,110 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:41:06,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731357666110"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731357666110"}]},"ts":"1731357666110"} 2024-11-11T20:41:06,114 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:41:06,115 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:41:06,116 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357666115"}]},"ts":"1731357666115"} 2024-11-11T20:41:06,118 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:41:06,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, ASSIGN}] 2024-11-11T20:41:06,120 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, ASSIGN 2024-11-11T20:41:06,121 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44271,1731357576703; forceNewPlan=false, retain=false 2024-11-11T20:41:06,272 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fe28f5753443ca2f28d81a625707f258, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:06,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, ASSIGN because future has completed 2024-11-11T20:41:06,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:41:06,431 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:41:06,431 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7752): Opening region: {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:41:06,432 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,432 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:41:06,432 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7794): checking encryption for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,432 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7797): checking classloading for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,434 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,437 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe28f5753443ca2f28d81a625707f258 columnFamilyName A 2024-11-11T20:41:06,437 DEBUG [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:06,438 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(327): Store=fe28f5753443ca2f28d81a625707f258/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:41:06,438 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,440 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe28f5753443ca2f28d81a625707f258 columnFamilyName B 2024-11-11T20:41:06,440 DEBUG [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:06,441 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(327): Store=fe28f5753443ca2f28d81a625707f258/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:41:06,441 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,443 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe28f5753443ca2f28d81a625707f258 columnFamilyName C 2024-11-11T20:41:06,443 DEBUG [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:06,443 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(327): Store=fe28f5753443ca2f28d81a625707f258/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:41:06,443 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1038): replaying wal for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,444 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,445 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,447 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1048): stopping wal replay for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,447 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1060): Cleaning up temporary data for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,447 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:41:06,449 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1093): writing seq id for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T20:41:06,451 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:41:06,452 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1114): Opened fe28f5753443ca2f28d81a625707f258; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59495333, jitterRate=-0.11345045268535614}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:41:06,452 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:06,453 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1006): Region open journal for fe28f5753443ca2f28d81a625707f258: Running coprocessor pre-open hook at 1731357666433Writing region info on filesystem at 1731357666433Initializing all the Stores at 1731357666434 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357666434Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357666434Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357666434Cleaning up temporary data from old regions at 1731357666447 (+13 ms)Running coprocessor post-open hooks at 1731357666452 (+5 ms)Region opened successfully at 1731357666453 (+1 ms) 2024-11-11T20:41:06,454 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., pid=101, masterSystemTime=1731357666425 2024-11-11T20:41:06,456 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:06,456 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:06,457 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fe28f5753443ca2f28d81a625707f258, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:06,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:41:06,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-11T20:41:06,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 in 187 msec 2024-11-11T20:41:06,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T20:41:06,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, ASSIGN in 343 msec 2024-11-11T20:41:06,465 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:41:06,466 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357666465"}]},"ts":"1731357666465"} 2024-11-11T20:41:06,467 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:41:06,468 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:41:06,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-11T20:41:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:41:07,441 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:07,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:07,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:07,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:07,445 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784592c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:07,447 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24df4f55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:07,449 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:07,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:07,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:41:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,35885,1731357576039 2024-11-11T20:41:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79626c85 2024-11-11T20:41:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:41:07,455 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:41:07,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:41:07,457 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:41:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:07,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742212_1388 (size=995) 2024-11-11T20:41:07,873 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T20:41:07,873 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T20:41:07,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:41:07,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, REOPEN/MOVE}] 2024-11-11T20:41:07,879 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, REOPEN/MOVE 2024-11-11T20:41:07,880 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=fe28f5753443ca2f28d81a625707f258, regionState=CLOSING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:07,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, REOPEN/MOVE because future has completed 2024-11-11T20:41:07,882 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:41:07,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:41:08,035 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(122): Close fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,035 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:41:08,035 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1722): Closing fe28f5753443ca2f28d81a625707f258, disabling compactions & flushes 2024-11-11T20:41:08,035 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,036 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,036 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. after waiting 0 ms 2024-11-11T20:41:08,036 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,043 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:41:08,044 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,044 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1676): Region close journal for fe28f5753443ca2f28d81a625707f258: Waiting for close lock at 1731357668035Running coprocessor pre-close hooks at 1731357668035Disabling compacts and flushes for region at 1731357668035Disabling writes for close at 1731357668036 (+1 ms)Writing region close event to WAL at 1731357668036Running coprocessor post-close hooks at 1731357668044 (+8 ms)Closed at 1731357668044 2024-11-11T20:41:08,044 WARN [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegionServer(3299): Not adding moved region record: fe28f5753443ca2f28d81a625707f258 to self. 2024-11-11T20:41:08,047 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(157): Closed fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,048 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=fe28f5753443ca2f28d81a625707f258, regionState=CLOSED 2024-11-11T20:41:08,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:41:08,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-11T20:41:08,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; CloseRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 in 170 msec 2024-11-11T20:41:08,056 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, REOPEN/MOVE; state=CLOSED, location=51ca66f7ee3c,44271,1731357576703; forceNewPlan=false, retain=true 2024-11-11T20:41:08,207 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=fe28f5753443ca2f28d81a625707f258, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, REOPEN/MOVE because future has completed 2024-11-11T20:41:08,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:41:08,368 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,368 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:41:08,369 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,369 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:41:08,369 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,369 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,371 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,371 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe28f5753443ca2f28d81a625707f258 columnFamilyName A 2024-11-11T20:41:08,372 DEBUG [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:08,372 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(327): Store=fe28f5753443ca2f28d81a625707f258/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:41:08,373 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,373 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe28f5753443ca2f28d81a625707f258 columnFamilyName B 2024-11-11T20:41:08,373 DEBUG [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:08,373 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(327): Store=fe28f5753443ca2f28d81a625707f258/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:41:08,374 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,374 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe28f5753443ca2f28d81a625707f258 columnFamilyName C 2024-11-11T20:41:08,374 DEBUG [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:08,374 INFO [StoreOpener-fe28f5753443ca2f28d81a625707f258-1 {}] regionserver.HStore(327): Store=fe28f5753443ca2f28d81a625707f258/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:41:08,374 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,375 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,376 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,376 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,376 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,377 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:41:08,378 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,378 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened fe28f5753443ca2f28d81a625707f258; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60742474, jitterRate=-0.09486660361289978}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:41:08,378 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,379 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for fe28f5753443ca2f28d81a625707f258: Running coprocessor pre-open hook at 1731357668369Writing region info on filesystem at 1731357668369Initializing all the Stores at 1731357668370 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357668370Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357668370Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357668370Cleaning up temporary data from old regions at 1731357668376 (+6 ms)Running coprocessor post-open hooks at 1731357668378 (+2 ms)Region opened successfully at 1731357668379 (+1 ms) 2024-11-11T20:41:08,379 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., pid=106, masterSystemTime=1731357668362 2024-11-11T20:41:08,381 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,381 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,381 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=fe28f5753443ca2f28d81a625707f258, regionState=OPEN, openSeqNum=5, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:41:08,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=104 2024-11-11T20:41:08,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=104, state=SUCCESS, hasLock=false; OpenRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 in 174 msec 2024-11-11T20:41:08,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T20:41:08,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, REOPEN/MOVE in 507 msec 2024-11-11T20:41:08,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-11T20:41:08,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-11T20:41:08,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 931 msec 2024-11-11T20:41:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-11-11T20:41:08,391 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:08,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,392 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,392 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e15df87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,393 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13f865ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,395 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a721f4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,396 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb50db6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,397 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d14de7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,398 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7802471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,400 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f150187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,401 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5cf57e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,402 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6176dea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,403 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d24c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,404 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,404 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,405 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d88265, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,406 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d1ec7dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,408 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424346e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,409 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591e1637, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,411 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca51703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,412 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b995153, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,414 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1977f314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,416 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3482e6e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:08,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:08,418 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29eda668, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:08,419 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:08,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c935b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:08,420 DEBUG [Thread-1739 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,420 DEBUG [Thread-1754 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,420 DEBUG [Thread-1756 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,420 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,421 DEBUG [Thread-1750 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,421 DEBUG [Thread-1745 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-11T20:41:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,422 DEBUG [Thread-1758 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,422 DEBUG [Thread-1747 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:41:08,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,423 DEBUG [Thread-1741 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,424 DEBUG [Thread-1752 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,424 DEBUG [Thread-1743 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,424 DEBUG [Thread-1747 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,424 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:08,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,425 DEBUG [Thread-1747 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,426 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:08,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:08,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:41:08,438 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:08,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:41:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111197cefb96694bd0a884b1c2f1247255_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357668436/Put/seqid=0 2024-11-11T20:41:08,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742213_1389 (size=14594) 2024-11-11T20:41:08,476 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:08,479 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111197cefb96694bd0a884b1c2f1247255_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111197cefb96694bd0a884b1c2f1247255_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:08,480 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/a1c7a27c061c4e43bacda7d05860e784, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:08,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/a1c7a27c061c4e43bacda7d05860e784 is 175, key is test_row_0/A:col10/1731357668436/Put/seqid=0 2024-11-11T20:41:08,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357728483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357728485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357728486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357728488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357728493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742214_1390 (size=39549) 2024-11-11T20:41:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:41:08,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T20:41:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:08,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357728611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357728611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357728611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357728611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357728611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T20:41:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:41:08,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357728821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357728821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357728821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357728821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357728821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:08,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T20:41:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:08,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:08,911 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/a1c7a27c061c4e43bacda7d05860e784 2024-11-11T20:41:08,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/490205f52d6046aa8c1cbaa1722d58fd is 50, key is test_row_0/B:col10/1731357668436/Put/seqid=0 2024-11-11T20:41:08,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742215_1391 (size=12001) 2024-11-11T20:41:08,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/490205f52d6046aa8c1cbaa1722d58fd 2024-11-11T20:41:08,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/2846e2cd878f4ea5b4f75c38a38993f3 is 50, key is test_row_0/C:col10/1731357668436/Put/seqid=0 2024-11-11T20:41:08,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742216_1392 (size=12001) 2024-11-11T20:41:08,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/2846e2cd878f4ea5b4f75c38a38993f3 2024-11-11T20:41:08,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/a1c7a27c061c4e43bacda7d05860e784 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a1c7a27c061c4e43bacda7d05860e784 2024-11-11T20:41:08,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a1c7a27c061c4e43bacda7d05860e784, entries=200, sequenceid=16, filesize=38.6 K 2024-11-11T20:41:08,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/490205f52d6046aa8c1cbaa1722d58fd as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/490205f52d6046aa8c1cbaa1722d58fd 2024-11-11T20:41:08,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/490205f52d6046aa8c1cbaa1722d58fd, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T20:41:08,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/2846e2cd878f4ea5b4f75c38a38993f3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/2846e2cd878f4ea5b4f75c38a38993f3 2024-11-11T20:41:08,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/2846e2cd878f4ea5b4f75c38a38993f3, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T20:41:08,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fe28f5753443ca2f28d81a625707f258 in 557ms, sequenceid=16, compaction requested=false 2024-11-11T20:41:08,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:09,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T20:41:09,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:09,039 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:41:09,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c766dad1d1a043f7b76173a82d472613_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357668468/Put/seqid=0 2024-11-11T20:41:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:41:09,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742217_1393 (size=12154) 2024-11-11T20:41:09,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:09,061 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c766dad1d1a043f7b76173a82d472613_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c766dad1d1a043f7b76173a82d472613_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:09,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/4747bf01bc134e3fbb8de0dad7f54328, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:09,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/4747bf01bc134e3fbb8de0dad7f54328 is 175, key is test_row_0/A:col10/1731357668468/Put/seqid=0 2024-11-11T20:41:09,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742218_1394 (size=30955) 2024-11-11T20:41:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:09,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:09,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357729138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357729138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357729147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357729147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357729147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357729251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357729251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357729251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357729261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357729261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357729461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357729461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357729461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,468 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/4747bf01bc134e3fbb8de0dad7f54328 2024-11-11T20:41:09,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/608385766ffe49739f68edcf82596f69 is 50, key is test_row_0/B:col10/1731357668468/Put/seqid=0 2024-11-11T20:41:09,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357729470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357729470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742219_1395 (size=12001) 2024-11-11T20:41:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:41:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357729771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357729771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357729771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357729780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357729780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:09,881 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/608385766ffe49739f68edcf82596f69 2024-11-11T20:41:09,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/84f0fcca643043f4a3ee161541763a57 is 50, key is test_row_0/C:col10/1731357668468/Put/seqid=0 2024-11-11T20:41:09,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742220_1396 (size=12001) 2024-11-11T20:41:10,052 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:41:10,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357730280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357730280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:10,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357730280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:10,292 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/84f0fcca643043f4a3ee161541763a57 2024-11-11T20:41:10,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357730290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:10,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/4747bf01bc134e3fbb8de0dad7f54328 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4747bf01bc134e3fbb8de0dad7f54328 2024-11-11T20:41:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:10,305 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4747bf01bc134e3fbb8de0dad7f54328, entries=150, sequenceid=40, filesize=30.2 K 2024-11-11T20:41:10,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/608385766ffe49739f68edcf82596f69 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/608385766ffe49739f68edcf82596f69 2024-11-11T20:41:10,311 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/608385766ffe49739f68edcf82596f69, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T20:41:10,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:10,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/84f0fcca643043f4a3ee161541763a57 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/84f0fcca643043f4a3ee161541763a57 2024-11-11T20:41:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357730301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:10,315 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/84f0fcca643043f4a3ee161541763a57, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T20:41:10,316 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for fe28f5753443ca2f28d81a625707f258 in 1277ms, sequenceid=40, compaction requested=false 2024-11-11T20:41:10,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:10,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:10,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-11T20:41:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-11T20:41:10,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T20:41:10,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8910 sec 2024-11-11T20:41:10,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8990 sec 2024-11-11T20:41:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:41:10,571 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:10,571 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-11T20:41:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:41:10,573 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:10,573 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:10,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:41:10,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T20:41:10,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:10,725 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:41:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119dbcd4f53955448a97935a50399d8863_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357669146/Put/seqid=0 2024-11-11T20:41:10,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742221_1397 (size=12154) 2024-11-11T20:41:10,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:10,747 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119dbcd4f53955448a97935a50399d8863_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119dbcd4f53955448a97935a50399d8863_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:10,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/96cfa0d12995484b8c32152119c916ac, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:10,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/96cfa0d12995484b8c32152119c916ac is 175, key is test_row_0/A:col10/1731357669146/Put/seqid=0 2024-11-11T20:41:10,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742222_1398 (size=30955) 2024-11-11T20:41:10,756 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/96cfa0d12995484b8c32152119c916ac 2024-11-11T20:41:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ff6686d7f1574ba8ba63afd3bed77c49 is 50, key is test_row_0/B:col10/1731357669146/Put/seqid=0 2024-11-11T20:41:10,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742223_1399 (size=12001) 2024-11-11T20:41:10,769 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ff6686d7f1574ba8ba63afd3bed77c49 2024-11-11T20:41:10,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/0d3f034f608f41e2969399b2931db8da is 50, key is test_row_0/C:col10/1731357669146/Put/seqid=0 2024-11-11T20:41:10,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742224_1400 (size=12001) 2024-11-11T20:41:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:41:11,201 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/0d3f034f608f41e2969399b2931db8da 2024-11-11T20:41:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:41:11,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/96cfa0d12995484b8c32152119c916ac as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/96cfa0d12995484b8c32152119c916ac 2024-11-11T20:41:11,208 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/96cfa0d12995484b8c32152119c916ac, entries=150, sequenceid=52, filesize=30.2 K 2024-11-11T20:41:11,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ff6686d7f1574ba8ba63afd3bed77c49 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ff6686d7f1574ba8ba63afd3bed77c49 2024-11-11T20:41:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,212 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ff6686d7f1574ba8ba63afd3bed77c49, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T20:41:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/0d3f034f608f41e2969399b2931db8da as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0d3f034f608f41e2969399b2931db8da 2024-11-11T20:41:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,216 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0d3f034f608f41e2969399b2931db8da, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T20:41:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,217 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for fe28f5753443ca2f28d81a625707f258 in 492ms, sequenceid=52, compaction requested=true 2024-11-11T20:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-11T20:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-11T20:41:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T20:41:11,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 645 msec 2024-11-11T20:41:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 649 msec 2024-11-11T20:41:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:11,405 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:11,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:41:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a3eb28a847ad40b2a2de2d09b1e0f195_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357671403/Put/seqid=0 2024-11-11T20:41:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742225_1401 (size=24358) 2024-11-11T20:41:11,468 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,473 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a3eb28a847ad40b2a2de2d09b1e0f195_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a3eb28a847ad40b2a2de2d09b1e0f195_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,474 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/6e3b4cdee5474d659bd752154a103378, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/6e3b4cdee5474d659bd752154a103378 is 175, key is test_row_0/A:col10/1731357671403/Put/seqid=0 2024-11-11T20:41:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742226_1402 (size=73995) 2024-11-11T20:41:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357731502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:11,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357731507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357731522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357731523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357731526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357731631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357731640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357731640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357731652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357731652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:41:11,712 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:11,712 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-11T20:41:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:41:11,717 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:11,718 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:11,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:41:11,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357731851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:41:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357731861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357731861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:11,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357731871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357731871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:11,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:11,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:11,897 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/6e3b4cdee5474d659bd752154a103378 2024-11-11T20:41:11,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/b26aeb1f16154492bb21bf7dc4ca16e2 is 50, key is test_row_0/B:col10/1731357671403/Put/seqid=0 2024-11-11T20:41:11,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742227_1403 (size=12001) 2024-11-11T20:41:12,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:41:12,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:12,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:12,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:41:12,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357732161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:41:12,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:12,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:12,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:12,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357732180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357732181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357732181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357732191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:41:12,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:12,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:12,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:12,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/b26aeb1f16154492bb21bf7dc4ca16e2 2024-11-11T20:41:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:41:12,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/23d13ae3cd3a4f0a90d38b1a4ed05bb1 is 50, key is test_row_0/C:col10/1731357671403/Put/seqid=0 2024-11-11T20:41:12,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742228_1404 (size=12001) 2024-11-11T20:41:12,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/23d13ae3cd3a4f0a90d38b1a4ed05bb1 2024-11-11T20:41:12,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/6e3b4cdee5474d659bd752154a103378 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6e3b4cdee5474d659bd752154a103378 2024-11-11T20:41:12,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6e3b4cdee5474d659bd752154a103378, entries=400, sequenceid=63, filesize=72.3 K 2024-11-11T20:41:12,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/b26aeb1f16154492bb21bf7dc4ca16e2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/b26aeb1f16154492bb21bf7dc4ca16e2 2024-11-11T20:41:12,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/b26aeb1f16154492bb21bf7dc4ca16e2, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T20:41:12,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/23d13ae3cd3a4f0a90d38b1a4ed05bb1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/23d13ae3cd3a4f0a90d38b1a4ed05bb1 2024-11-11T20:41:12,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/23d13ae3cd3a4f0a90d38b1a4ed05bb1, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T20:41:12,429 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T20:41:12,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fe28f5753443ca2f28d81a625707f258 in 1025ms, sequenceid=63, compaction requested=true 2024-11-11T20:41:12,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:12,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:12,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:12,431 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:12,431 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:12,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:12,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:12,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:12,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:12,433 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 175454 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:12,433 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/A is initiating minor compaction (all files) 2024-11-11T20:41:12,433 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/A in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,433 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a1c7a27c061c4e43bacda7d05860e784, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4747bf01bc134e3fbb8de0dad7f54328, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/96cfa0d12995484b8c32152119c916ac, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6e3b4cdee5474d659bd752154a103378] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=171.3 K 2024-11-11T20:41:12,433 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,433 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a1c7a27c061c4e43bacda7d05860e784, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4747bf01bc134e3fbb8de0dad7f54328, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/96cfa0d12995484b8c32152119c916ac, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6e3b4cdee5474d659bd752154a103378] 2024-11-11T20:41:12,434 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:12,434 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/B is initiating minor compaction (all files) 2024-11-11T20:41:12,434 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/B in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,434 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/490205f52d6046aa8c1cbaa1722d58fd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/608385766ffe49739f68edcf82596f69, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ff6686d7f1574ba8ba63afd3bed77c49, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/b26aeb1f16154492bb21bf7dc4ca16e2] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=46.9 K 2024-11-11T20:41:12,434 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 490205f52d6046aa8c1cbaa1722d58fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731357668431 2024-11-11T20:41:12,434 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1c7a27c061c4e43bacda7d05860e784, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731357668431 2024-11-11T20:41:12,435 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 608385766ffe49739f68edcf82596f69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731357668468 2024-11-11T20:41:12,435 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4747bf01bc134e3fbb8de0dad7f54328, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731357668468 2024-11-11T20:41:12,435 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ff6686d7f1574ba8ba63afd3bed77c49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731357669136 2024-11-11T20:41:12,435 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96cfa0d12995484b8c32152119c916ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731357669136 2024-11-11T20:41:12,435 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b26aeb1f16154492bb21bf7dc4ca16e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731357671378 2024-11-11T20:41:12,436 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e3b4cdee5474d659bd752154a103378, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731357671375 2024-11-11T20:41:12,460 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:12,461 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/a2cbc01393024b2cb3ff86c93058b95d is 50, key is test_row_0/B:col10/1731357671403/Put/seqid=0 2024-11-11T20:41:12,465 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:12,474 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111f3908721d74b427fa16fe916aed19be4_fe28f5753443ca2f28d81a625707f258 store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:12,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742229_1405 (size=12139) 2024-11-11T20:41:12,478 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111f3908721d74b427fa16fe916aed19be4_fe28f5753443ca2f28d81a625707f258, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:12,478 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f3908721d74b427fa16fe916aed19be4_fe28f5753443ca2f28d81a625707f258 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:12,482 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/a2cbc01393024b2cb3ff86c93058b95d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a2cbc01393024b2cb3ff86c93058b95d 2024-11-11T20:41:12,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742230_1406 (size=4469) 2024-11-11T20:41:12,484 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#A#compaction#346 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:12,484 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/d968f98a8b194864a4f853565b31f68b is 175, key is test_row_0/A:col10/1731357671403/Put/seqid=0 2024-11-11T20:41:12,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:41:12,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,485 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T20:41:12,493 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe28f5753443ca2f28d81a625707f258/B of fe28f5753443ca2f28d81a625707f258 into a2cbc01393024b2cb3ff86c93058b95d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:12,494 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:12,494 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/B, priority=12, startTime=1731357672431; duration=0sec 2024-11-11T20:41:12,494 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:12,494 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:B 2024-11-11T20:41:12,494 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:12,495 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:12,495 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/C is initiating minor compaction (all files) 2024-11-11T20:41:12,495 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/C in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:12,495 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/2846e2cd878f4ea5b4f75c38a38993f3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/84f0fcca643043f4a3ee161541763a57, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0d3f034f608f41e2969399b2931db8da, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/23d13ae3cd3a4f0a90d38b1a4ed05bb1] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=46.9 K 2024-11-11T20:41:12,495 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2846e2cd878f4ea5b4f75c38a38993f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731357668431 2024-11-11T20:41:12,496 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 84f0fcca643043f4a3ee161541763a57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731357668468 2024-11-11T20:41:12,497 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d3f034f608f41e2969399b2931db8da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731357669136 2024-11-11T20:41:12,500 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d13ae3cd3a4f0a90d38b1a4ed05bb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731357671378 2024-11-11T20:41:12,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742231_1407 (size=31093) 2024-11-11T20:41:12,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113e54a3772dcb429692b661d542519854_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357671521/Put/seqid=0 2024-11-11T20:41:12,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742232_1408 (size=12154) 2024-11-11T20:41:12,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:12,522 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113e54a3772dcb429692b661d542519854_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113e54a3772dcb429692b661d542519854_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:12,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/239839805fa2407e93c6fc4fd16125c6, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:12,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/239839805fa2407e93c6fc4fd16125c6 is 175, key is test_row_0/A:col10/1731357671521/Put/seqid=0 2024-11-11T20:41:12,531 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#C#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:12,532 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/d02d5fc66acb4890b19dcf47b1924678 is 50, key is test_row_0/C:col10/1731357671403/Put/seqid=0 2024-11-11T20:41:12,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742233_1409 (size=30955) 2024-11-11T20:41:12,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742234_1410 (size=12139) 2024-11-11T20:41:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:12,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:12,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357732702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357732732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357732732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357732732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357732733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357732840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:41:12,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357732851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357732851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357732851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357732851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:12,904 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/d968f98a8b194864a4f853565b31f68b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d968f98a8b194864a4f853565b31f68b 2024-11-11T20:41:12,908 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe28f5753443ca2f28d81a625707f258/A of fe28f5753443ca2f28d81a625707f258 into d968f98a8b194864a4f853565b31f68b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:12,908 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:12,908 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/A, priority=12, startTime=1731357672430; duration=0sec 2024-11-11T20:41:12,908 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:12,908 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:A 2024-11-11T20:41:12,944 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/239839805fa2407e93c6fc4fd16125c6 2024-11-11T20:41:12,949 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/d02d5fc66acb4890b19dcf47b1924678 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d02d5fc66acb4890b19dcf47b1924678 2024-11-11T20:41:12,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/a90f2aa0a60b4bf89d5045ce39a79f5c is 50, key is test_row_0/B:col10/1731357671521/Put/seqid=0 2024-11-11T20:41:12,956 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe28f5753443ca2f28d81a625707f258/C of fe28f5753443ca2f28d81a625707f258 into d02d5fc66acb4890b19dcf47b1924678(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:12,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742235_1411 (size=12001) 2024-11-11T20:41:12,956 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:12,956 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/C, priority=12, startTime=1731357672431; duration=0sec 2024-11-11T20:41:12,956 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:12,956 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:C 2024-11-11T20:41:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357733051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357733071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357733071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357733071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357733071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,356 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/a90f2aa0a60b4bf89d5045ce39a79f5c 2024-11-11T20:41:13,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/a62ec120d491486e8bd05e3563519706 is 50, key is test_row_0/C:col10/1731357671521/Put/seqid=0 2024-11-11T20:41:13,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742236_1412 (size=12001) 2024-11-11T20:41:13,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357733361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357733381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357733390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357733390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357733390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,765 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/a62ec120d491486e8bd05e3563519706 2024-11-11T20:41:13,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/239839805fa2407e93c6fc4fd16125c6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/239839805fa2407e93c6fc4fd16125c6 2024-11-11T20:41:13,772 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/239839805fa2407e93c6fc4fd16125c6, entries=150, sequenceid=88, filesize=30.2 K 2024-11-11T20:41:13,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/a90f2aa0a60b4bf89d5045ce39a79f5c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a90f2aa0a60b4bf89d5045ce39a79f5c 2024-11-11T20:41:13,775 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a90f2aa0a60b4bf89d5045ce39a79f5c, entries=150, sequenceid=88, filesize=11.7 K 2024-11-11T20:41:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/a62ec120d491486e8bd05e3563519706 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a62ec120d491486e8bd05e3563519706 2024-11-11T20:41:13,778 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a62ec120d491486e8bd05e3563519706, entries=150, sequenceid=88, filesize=11.7 K 2024-11-11T20:41:13,779 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fe28f5753443ca2f28d81a625707f258 in 1294ms, sequenceid=88, compaction requested=false 2024-11-11T20:41:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-11T20:41:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-11T20:41:13,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T20:41:13,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0620 sec 2024-11-11T20:41:13,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.0700 sec 2024-11-11T20:41:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:41:13,861 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:13,861 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-11T20:41:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:41:13,863 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:13,864 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:13,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:13,884 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:13,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:41:13,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b36e1c41a7764f59b40ab74bd8806feb_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357672701/Put/seqid=0 2024-11-11T20:41:13,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742237_1413 (size=14594) 2024-11-11T20:41:13,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:13,898 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b36e1c41a7764f59b40ab74bd8806feb_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b36e1c41a7764f59b40ab74bd8806feb_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:13,899 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f47e8bb5b08644c088017a7145a406c2, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:13,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f47e8bb5b08644c088017a7145a406c2 is 175, key is test_row_0/A:col10/1731357672701/Put/seqid=0 2024-11-11T20:41:13,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742238_1414 (size=39549) 2024-11-11T20:41:13,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f47e8bb5b08644c088017a7145a406c2 2024-11-11T20:41:13,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ac6b2af92f7b45889a7cab7795e06c9b is 50, key is test_row_0/B:col10/1731357672701/Put/seqid=0 2024-11-11T20:41:13,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742239_1415 (size=12001) 2024-11-11T20:41:13,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357733933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357733936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357733937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357733943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357733944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:41:14,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T20:41:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357734050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357734050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357734050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357734050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357734061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T20:41:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:41:14,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357734260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357734260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357734271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357734271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357734281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ac6b2af92f7b45889a7cab7795e06c9b 2024-11-11T20:41:14,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T20:41:14,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:14,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/d7c06f6f092444a8b173c21c17bb2297 is 50, key is test_row_0/C:col10/1731357672701/Put/seqid=0 2024-11-11T20:41:14,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742240_1416 (size=12001) 2024-11-11T20:41:14,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T20:41:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:41:14,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357734570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357734570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357734580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357734580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357734591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:14,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T20:41:14,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:14,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:14,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/d7c06f6f092444a8b173c21c17bb2297 2024-11-11T20:41:14,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f47e8bb5b08644c088017a7145a406c2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f47e8bb5b08644c088017a7145a406c2 2024-11-11T20:41:14,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f47e8bb5b08644c088017a7145a406c2, entries=200, sequenceid=103, filesize=38.6 K 2024-11-11T20:41:14,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ac6b2af92f7b45889a7cab7795e06c9b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ac6b2af92f7b45889a7cab7795e06c9b 2024-11-11T20:41:14,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ac6b2af92f7b45889a7cab7795e06c9b, entries=150, sequenceid=103, filesize=11.7 K 2024-11-11T20:41:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/d7c06f6f092444a8b173c21c17bb2297 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d7c06f6f092444a8b173c21c17bb2297 2024-11-11T20:41:14,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d7c06f6f092444a8b173c21c17bb2297, entries=150, sequenceid=103, filesize=11.7 K 2024-11-11T20:41:14,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fe28f5753443ca2f28d81a625707f258 in 866ms, sequenceid=103, compaction requested=true 2024-11-11T20:41:14,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:14,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:14,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:14,750 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:14,751 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:14,752 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:14,752 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/A is initiating minor compaction (all files) 2024-11-11T20:41:14,752 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/A in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,752 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d968f98a8b194864a4f853565b31f68b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/239839805fa2407e93c6fc4fd16125c6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f47e8bb5b08644c088017a7145a406c2] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=99.2 K 2024-11-11T20:41:14,752 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,752 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d968f98a8b194864a4f853565b31f68b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/239839805fa2407e93c6fc4fd16125c6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f47e8bb5b08644c088017a7145a406c2] 2024-11-11T20:41:14,753 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:14,753 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/B is initiating minor compaction (all files) 2024-11-11T20:41:14,753 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/B in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,753 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a2cbc01393024b2cb3ff86c93058b95d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a90f2aa0a60b4bf89d5045ce39a79f5c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ac6b2af92f7b45889a7cab7795e06c9b] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=35.3 K 2024-11-11T20:41:14,753 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d968f98a8b194864a4f853565b31f68b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731357671378 2024-11-11T20:41:14,753 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a2cbc01393024b2cb3ff86c93058b95d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731357671378 2024-11-11T20:41:14,754 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 239839805fa2407e93c6fc4fd16125c6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731357671505 2024-11-11T20:41:14,754 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a90f2aa0a60b4bf89d5045ce39a79f5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731357671505 2024-11-11T20:41:14,754 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f47e8bb5b08644c088017a7145a406c2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731357672701 2024-11-11T20:41:14,754 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6b2af92f7b45889a7cab7795e06c9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731357672701 2024-11-11T20:41:14,761 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:14,763 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:14,763 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/1afd0743359f4056986de7de0029237b is 50, key is test_row_0/B:col10/1731357672701/Put/seqid=0 2024-11-11T20:41:14,765 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411114db1ee76e12e4aafaca6f153b2225a10_fe28f5753443ca2f28d81a625707f258 store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:14,768 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411114db1ee76e12e4aafaca6f153b2225a10_fe28f5753443ca2f28d81a625707f258, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:14,768 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114db1ee76e12e4aafaca6f153b2225a10_fe28f5753443ca2f28d81a625707f258 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:14,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742241_1417 (size=12241) 2024-11-11T20:41:14,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742242_1418 (size=4469) 2024-11-11T20:41:14,778 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/1afd0743359f4056986de7de0029237b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/1afd0743359f4056986de7de0029237b 2024-11-11T20:41:14,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T20:41:14,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,780 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:41:14,780 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#A#compaction#355 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:14,781 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f89c7c54cc9046d89f6f940d867b0aaf is 175, key is test_row_0/A:col10/1731357672701/Put/seqid=0 2024-11-11T20:41:14,784 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/B of fe28f5753443ca2f28d81a625707f258 into 1afd0743359f4056986de7de0029237b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:14,784 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:14,784 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/B, priority=13, startTime=1731357674750; duration=0sec 2024-11-11T20:41:14,784 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:14,784 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:B 2024-11-11T20:41:14,784 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:14,784 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:14,784 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/C is initiating minor compaction (all files) 2024-11-11T20:41:14,785 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/C in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:14,785 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d02d5fc66acb4890b19dcf47b1924678, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a62ec120d491486e8bd05e3563519706, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d7c06f6f092444a8b173c21c17bb2297] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=35.3 K 2024-11-11T20:41:14,785 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d02d5fc66acb4890b19dcf47b1924678, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731357671378 2024-11-11T20:41:14,785 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a62ec120d491486e8bd05e3563519706, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731357671505 2024-11-11T20:41:14,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742243_1419 (size=31195) 2024-11-11T20:41:14,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111faa457d487424064b55a9fdbf6c787ab_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357673935/Put/seqid=0 2024-11-11T20:41:14,788 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c06f6f092444a8b173c21c17bb2297, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731357672701 2024-11-11T20:41:14,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742244_1420 (size=12154) 2024-11-11T20:41:14,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:14,796 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111faa457d487424064b55a9fdbf6c787ab_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111faa457d487424064b55a9fdbf6c787ab_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:14,797 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:14,797 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/787c6219251a4c82b636a686748b4c18 is 50, key is test_row_0/C:col10/1731357672701/Put/seqid=0 2024-11-11T20:41:14,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/5343fffdfef148b3a4a6811d0f8f9688, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:14,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/5343fffdfef148b3a4a6811d0f8f9688 is 175, key is test_row_0/A:col10/1731357673935/Put/seqid=0 2024-11-11T20:41:14,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742245_1421 (size=12241) 2024-11-11T20:41:14,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742246_1422 (size=30955) 2024-11-11T20:41:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:41:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:15,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:15,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357735104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357735105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357735105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357735109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357735110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,193 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f89c7c54cc9046d89f6f940d867b0aaf as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f89c7c54cc9046d89f6f940d867b0aaf 2024-11-11T20:41:15,197 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/A of fe28f5753443ca2f28d81a625707f258 into f89c7c54cc9046d89f6f940d867b0aaf(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:15,197 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:15,197 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/A, priority=13, startTime=1731357674750; duration=0sec 2024-11-11T20:41:15,197 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:15,197 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:A 2024-11-11T20:41:15,206 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/787c6219251a4c82b636a686748b4c18 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/787c6219251a4c82b636a686748b4c18 2024-11-11T20:41:15,207 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/5343fffdfef148b3a4a6811d0f8f9688 2024-11-11T20:41:15,211 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/C of fe28f5753443ca2f28d81a625707f258 into 787c6219251a4c82b636a686748b4c18(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:15,211 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:15,211 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/C, priority=13, startTime=1731357674751; duration=0sec 2024-11-11T20:41:15,211 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:15,211 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:C 2024-11-11T20:41:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/957b2be6d48c40de8a57b5eb5856963e is 50, key is test_row_0/B:col10/1731357673935/Put/seqid=0 2024-11-11T20:41:15,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742247_1423 (size=12001) 2024-11-11T20:41:15,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357735220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357735221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357735221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357735221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357735221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357735430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357735431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357735431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357735431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357735431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,619 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/957b2be6d48c40de8a57b5eb5856963e 2024-11-11T20:41:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/4e598fdef2844637a1d4875d07bfba92 is 50, key is test_row_0/C:col10/1731357673935/Put/seqid=0 2024-11-11T20:41:15,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742248_1424 (size=12001) 2024-11-11T20:41:15,629 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/4e598fdef2844637a1d4875d07bfba92 2024-11-11T20:41:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/5343fffdfef148b3a4a6811d0f8f9688 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/5343fffdfef148b3a4a6811d0f8f9688 2024-11-11T20:41:15,636 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/5343fffdfef148b3a4a6811d0f8f9688, entries=150, sequenceid=127, filesize=30.2 K 2024-11-11T20:41:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/957b2be6d48c40de8a57b5eb5856963e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/957b2be6d48c40de8a57b5eb5856963e 2024-11-11T20:41:15,639 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/957b2be6d48c40de8a57b5eb5856963e, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T20:41:15,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/4e598fdef2844637a1d4875d07bfba92 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/4e598fdef2844637a1d4875d07bfba92 2024-11-11T20:41:15,643 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/4e598fdef2844637a1d4875d07bfba92, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T20:41:15,644 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for fe28f5753443ca2f28d81a625707f258 in 864ms, sequenceid=127, compaction requested=false 2024-11-11T20:41:15,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:15,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:15,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-11T20:41:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-11T20:41:15,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T20:41:15,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-11-11T20:41:15,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.7870 sec 2024-11-11T20:41:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:15,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:15,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T20:41:15,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ca05fba6e7d64fc08feb200ef150b242_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357675109/Put/seqid=0 2024-11-11T20:41:15,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742249_1425 (size=14794) 2024-11-11T20:41:15,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357735770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357735775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357735778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357735779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357735780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357735891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357735891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357735891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357735891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357735891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:41:16,010 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:16,011 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-11T20:41:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:41:16,012 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:16,012 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:16,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:16,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357736111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357736111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357736111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357736111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357736111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:41:16,163 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:16,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:16,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:16,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ca05fba6e7d64fc08feb200ef150b242_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ca05fba6e7d64fc08feb200ef150b242_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:16,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/8ab06d608cd647cc9a49144a7095d38c, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:16,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/8ab06d608cd647cc9a49144a7095d38c is 175, key is test_row_0/A:col10/1731357675109/Put/seqid=0 2024-11-11T20:41:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742250_1426 (size=39749) 2024-11-11T20:41:16,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:16,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:16,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:41:16,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357736421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357736421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357736421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357736421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357736421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T20:41:16,451 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T20:41:16,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:16,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:16,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,573 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/8ab06d608cd647cc9a49144a7095d38c 2024-11-11T20:41:16,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/f583e4ae41ca491aa0e42b12fe76d906 is 50, key is test_row_0/B:col10/1731357675109/Put/seqid=0 2024-11-11T20:41:16,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742251_1427 (size=12151) 2024-11-11T20:41:16,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:16,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:16,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:41:16,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:16,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:16,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:16,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:16,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:16,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357736930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357736931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357736931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357736931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357736941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:16,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/f583e4ae41ca491aa0e42b12fe76d906 2024-11-11T20:41:16,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/ab553dad304749a0aca2b79f81dfeb0c is 50, key is test_row_0/C:col10/1731357675109/Put/seqid=0 2024-11-11T20:41:16,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742252_1428 (size=12151) 2024-11-11T20:41:17,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:41:17,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:17,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:17,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:17,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:17,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:17,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/ab553dad304749a0aca2b79f81dfeb0c 2024-11-11T20:41:17,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/8ab06d608cd647cc9a49144a7095d38c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/8ab06d608cd647cc9a49144a7095d38c 2024-11-11T20:41:17,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/8ab06d608cd647cc9a49144a7095d38c, entries=200, sequenceid=143, filesize=38.8 K 2024-11-11T20:41:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/f583e4ae41ca491aa0e42b12fe76d906 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f583e4ae41ca491aa0e42b12fe76d906 2024-11-11T20:41:17,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f583e4ae41ca491aa0e42b12fe76d906, entries=150, sequenceid=143, filesize=11.9 K 2024-11-11T20:41:17,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/ab553dad304749a0aca2b79f81dfeb0c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/ab553dad304749a0aca2b79f81dfeb0c 2024-11-11T20:41:17,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/ab553dad304749a0aca2b79f81dfeb0c, entries=150, sequenceid=143, filesize=11.9 K 2024-11-11T20:41:17,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fe28f5753443ca2f28d81a625707f258 in 1683ms, sequenceid=143, compaction requested=true 2024-11-11T20:41:17,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:17,430 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:17,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:17,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:17,430 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:17,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:17,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:17,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:17,431 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:17,431 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/A is initiating minor compaction (all files) 2024-11-11T20:41:17,431 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/A in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,431 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f89c7c54cc9046d89f6f940d867b0aaf, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/5343fffdfef148b3a4a6811d0f8f9688, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/8ab06d608cd647cc9a49144a7095d38c] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=99.5 K 2024-11-11T20:41:17,431 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,431 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f89c7c54cc9046d89f6f940d867b0aaf, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/5343fffdfef148b3a4a6811d0f8f9688, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/8ab06d608cd647cc9a49144a7095d38c] 2024-11-11T20:41:17,431 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:17,431 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/B is initiating minor compaction (all files) 2024-11-11T20:41:17,431 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/B in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,432 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/1afd0743359f4056986de7de0029237b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/957b2be6d48c40de8a57b5eb5856963e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f583e4ae41ca491aa0e42b12fe76d906] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=35.5 K 2024-11-11T20:41:17,432 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f89c7c54cc9046d89f6f940d867b0aaf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731357672701 2024-11-11T20:41:17,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:17,432 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5343fffdfef148b3a4a6811d0f8f9688, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731357673931 2024-11-11T20:41:17,432 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1afd0743359f4056986de7de0029237b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731357672701 2024-11-11T20:41:17,433 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 957b2be6d48c40de8a57b5eb5856963e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731357673931 2024-11-11T20:41:17,433 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab06d608cd647cc9a49144a7095d38c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731357675098 2024-11-11T20:41:17,433 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f583e4ae41ca491aa0e42b12fe76d906, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731357675098 2024-11-11T20:41:17,472 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:17,472 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ccb2d4292fd84f5a9e2c6fbbb684f9b1 is 50, key is test_row_0/B:col10/1731357675109/Put/seqid=0 2024-11-11T20:41:17,474 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:17,478 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110d6f7190df314f9aa8a123c9d42c69bb_fe28f5753443ca2f28d81a625707f258 store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:17,480 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110d6f7190df314f9aa8a123c9d42c69bb_fe28f5753443ca2f28d81a625707f258, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:17,481 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110d6f7190df314f9aa8a123c9d42c69bb_fe28f5753443ca2f28d81a625707f258 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:17,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742253_1429 (size=12493) 2024-11-11T20:41:17,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742254_1430 (size=4469) 2024-11-11T20:41:17,538 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#A#compaction#364 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:17,538 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/a4ba48a9f7b14ea3a6d87df7499c51a6 is 175, key is test_row_0/A:col10/1731357675109/Put/seqid=0 2024-11-11T20:41:17,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:41:17,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,547 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T20:41:17,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742255_1431 (size=31447) 2024-11-11T20:41:17,576 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/a4ba48a9f7b14ea3a6d87df7499c51a6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a4ba48a9f7b14ea3a6d87df7499c51a6 2024-11-11T20:41:17,586 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/A of fe28f5753443ca2f28d81a625707f258 into a4ba48a9f7b14ea3a6d87df7499c51a6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:17,586 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:17,586 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/A, priority=13, startTime=1731357677429; duration=0sec 2024-11-11T20:41:17,586 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:17,586 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:A 2024-11-11T20:41:17,586 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:17,590 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:17,590 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/C is initiating minor compaction (all files) 2024-11-11T20:41:17,590 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/C in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:17,590 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/787c6219251a4c82b636a686748b4c18, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/4e598fdef2844637a1d4875d07bfba92, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/ab553dad304749a0aca2b79f81dfeb0c] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=35.5 K 2024-11-11T20:41:17,590 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787c6219251a4c82b636a686748b4c18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731357672701 2024-11-11T20:41:17,591 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e598fdef2844637a1d4875d07bfba92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731357673931 2024-11-11T20:41:17,591 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab553dad304749a0aca2b79f81dfeb0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731357675098 2024-11-11T20:41:17,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111be135255a4974d07bf2603ef2c8980d9_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357675774/Put/seqid=0 2024-11-11T20:41:17,623 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:17,624 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/b4060a9c84814690979c82740cb603b3 is 50, key is test_row_0/C:col10/1731357675109/Put/seqid=0 2024-11-11T20:41:17,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742256_1432 (size=12304) 2024-11-11T20:41:17,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742257_1433 (size=12493) 2024-11-11T20:41:17,662 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/b4060a9c84814690979c82740cb603b3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4060a9c84814690979c82740cb603b3 2024-11-11T20:41:17,669 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/C of fe28f5753443ca2f28d81a625707f258 into b4060a9c84814690979c82740cb603b3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:17,669 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:17,669 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/C, priority=13, startTime=1731357677430; duration=0sec 2024-11-11T20:41:17,669 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:17,669 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:C 2024-11-11T20:41:17,932 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ccb2d4292fd84f5a9e2c6fbbb684f9b1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ccb2d4292fd84f5a9e2c6fbbb684f9b1 2024-11-11T20:41:17,937 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/B of fe28f5753443ca2f28d81a625707f258 into ccb2d4292fd84f5a9e2c6fbbb684f9b1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:17,937 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:17,937 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/B, priority=13, startTime=1731357677430; duration=0sec 2024-11-11T20:41:17,937 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:17,937 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:B 2024-11-11T20:41:17,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:17,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357737967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:17,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357737968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:17,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357737975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:17,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357737977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:17,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357737980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:18,068 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111be135255a4974d07bf2603ef2c8980d9_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111be135255a4974d07bf2603ef2c8980d9_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/dc1b250f7d744c6886a634fa7c777d56, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:18,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/dc1b250f7d744c6886a634fa7c777d56 is 175, key is test_row_0/A:col10/1731357675774/Put/seqid=0 2024-11-11T20:41:18,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357738090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357738090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357738101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357738101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357738101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742258_1434 (size=31105) 2024-11-11T20:41:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:41:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357738311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357738311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357738320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357738321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357738321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,515 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/dc1b250f7d744c6886a634fa7c777d56 2024-11-11T20:41:18,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d3d31050028144bea5dd467ced52f101 is 50, key is test_row_0/B:col10/1731357675774/Put/seqid=0 2024-11-11T20:41:18,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742259_1435 (size=12151) 2024-11-11T20:41:18,544 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d3d31050028144bea5dd467ced52f101 2024-11-11T20:41:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/e36640d337e644668fa945f0464b65e0 is 50, key is test_row_0/C:col10/1731357675774/Put/seqid=0 2024-11-11T20:41:18,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742260_1436 (size=12151) 2024-11-11T20:41:18,585 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/e36640d337e644668fa945f0464b65e0 2024-11-11T20:41:18,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/dc1b250f7d744c6886a634fa7c777d56 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/dc1b250f7d744c6886a634fa7c777d56 2024-11-11T20:41:18,594 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/dc1b250f7d744c6886a634fa7c777d56, entries=150, sequenceid=166, filesize=30.4 K 2024-11-11T20:41:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d3d31050028144bea5dd467ced52f101 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3d31050028144bea5dd467ced52f101 2024-11-11T20:41:18,601 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3d31050028144bea5dd467ced52f101, entries=150, sequenceid=166, filesize=11.9 K 2024-11-11T20:41:18,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/e36640d337e644668fa945f0464b65e0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/e36640d337e644668fa945f0464b65e0 2024-11-11T20:41:18,612 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/e36640d337e644668fa945f0464b65e0, entries=150, sequenceid=166, filesize=11.9 K 2024-11-11T20:41:18,613 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for fe28f5753443ca2f28d81a625707f258 in 1066ms, sequenceid=166, compaction requested=false 2024-11-11T20:41:18,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:18,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:18,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-11T20:41:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-11T20:41:18,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T20:41:18,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6020 sec 2024-11-11T20:41:18,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.6070 sec 2024-11-11T20:41:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:18,645 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:18,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T20:41:18,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aeb664e7c03642ca9e59f32c0c1bb22e_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357678643/Put/seqid=0 2024-11-11T20:41:18,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742261_1437 (size=12304) 2024-11-11T20:41:18,712 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:18,718 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aeb664e7c03642ca9e59f32c0c1bb22e_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aeb664e7c03642ca9e59f32c0c1bb22e_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:18,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/7bba27a67d8a463b9c9739ea35712eb5, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:18,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/7bba27a67d8a463b9c9739ea35712eb5 is 175, key is test_row_0/A:col10/1731357678643/Put/seqid=0 2024-11-11T20:41:18,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742262_1438 (size=31105) 2024-11-11T20:41:18,731 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/7bba27a67d8a463b9c9739ea35712eb5 2024-11-11T20:41:18,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ee1cf7e02bbf4dccbbb11ca4a7afa102 is 50, key is test_row_0/B:col10/1731357678643/Put/seqid=0 2024-11-11T20:41:18,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357738754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357738754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357738756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357738757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357738771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742263_1439 (size=12151) 2024-11-11T20:41:18,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ee1cf7e02bbf4dccbbb11ca4a7afa102 2024-11-11T20:41:18,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/b4f88a94ee3248f8abde383d06b5f182 is 50, key is test_row_0/C:col10/1731357678643/Put/seqid=0 2024-11-11T20:41:18,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742264_1440 (size=12151) 2024-11-11T20:41:18,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/b4f88a94ee3248f8abde383d06b5f182 2024-11-11T20:41:18,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/7bba27a67d8a463b9c9739ea35712eb5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/7bba27a67d8a463b9c9739ea35712eb5 2024-11-11T20:41:18,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/7bba27a67d8a463b9c9739ea35712eb5, entries=150, sequenceid=186, filesize=30.4 K 2024-11-11T20:41:18,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/ee1cf7e02bbf4dccbbb11ca4a7afa102 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ee1cf7e02bbf4dccbbb11ca4a7afa102 2024-11-11T20:41:18,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ee1cf7e02bbf4dccbbb11ca4a7afa102, entries=150, sequenceid=186, filesize=11.9 K 2024-11-11T20:41:18,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/b4f88a94ee3248f8abde383d06b5f182 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4f88a94ee3248f8abde383d06b5f182 2024-11-11T20:41:18,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4f88a94ee3248f8abde383d06b5f182, entries=150, sequenceid=186, filesize=11.9 K 2024-11-11T20:41:18,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for fe28f5753443ca2f28d81a625707f258 in 193ms, sequenceid=186, compaction requested=true 2024-11-11T20:41:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:18,838 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:18,838 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:18,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:18,839 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:18,839 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/A is initiating minor compaction (all files) 2024-11-11T20:41:18,839 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/A in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:18,839 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a4ba48a9f7b14ea3a6d87df7499c51a6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/dc1b250f7d744c6886a634fa7c777d56, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/7bba27a67d8a463b9c9739ea35712eb5] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=91.5 K 2024-11-11T20:41:18,839 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:18,839 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a4ba48a9f7b14ea3a6d87df7499c51a6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/dc1b250f7d744c6886a634fa7c777d56, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/7bba27a67d8a463b9c9739ea35712eb5] 2024-11-11T20:41:18,839 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:18,839 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/B is initiating minor compaction (all files) 2024-11-11T20:41:18,839 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/B in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:18,839 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ccb2d4292fd84f5a9e2c6fbbb684f9b1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3d31050028144bea5dd467ced52f101, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ee1cf7e02bbf4dccbbb11ca4a7afa102] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=35.9 K 2024-11-11T20:41:18,840 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ba48a9f7b14ea3a6d87df7499c51a6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731357675098 2024-11-11T20:41:18,840 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ccb2d4292fd84f5a9e2c6fbbb684f9b1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731357675098 2024-11-11T20:41:18,840 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc1b250f7d744c6886a634fa7c777d56, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731357675774 2024-11-11T20:41:18,840 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d31050028144bea5dd467ced52f101, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731357675774 2024-11-11T20:41:18,840 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bba27a67d8a463b9c9739ea35712eb5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731357677976 2024-11-11T20:41:18,840 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1cf7e02bbf4dccbbb11ca4a7afa102, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731357677976 2024-11-11T20:41:18,862 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:18,863 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#B#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:18,863 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d107fc96e5344cac8d8540c07368bce6 is 50, key is test_row_0/B:col10/1731357678643/Put/seqid=0 2024-11-11T20:41:18,869 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c678bef5b6c9433684b4165d3ffc3c84_fe28f5753443ca2f28d81a625707f258 store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:18,872 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c678bef5b6c9433684b4165d3ffc3c84_fe28f5753443ca2f28d81a625707f258, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:18,872 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c678bef5b6c9433684b4165d3ffc3c84_fe28f5753443ca2f28d81a625707f258 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:18,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742265_1441 (size=12595) 2024-11-11T20:41:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:18,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T20:41:18,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742266_1442 (size=4469) 2024-11-11T20:41:18,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357738930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111129555c110dfc43eb91be03f52ba1cfcf_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357678753/Put/seqid=0 2024-11-11T20:41:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357738930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357738946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357738948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357738949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:18,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742267_1443 (size=14794) 2024-11-11T20:41:18,977 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:18,982 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111129555c110dfc43eb91be03f52ba1cfcf_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111129555c110dfc43eb91be03f52ba1cfcf_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:18,983 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/415512790d5a499a9aaffae96fd8f87d, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:18,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/415512790d5a499a9aaffae96fd8f87d is 175, key is test_row_0/A:col10/1731357678753/Put/seqid=0 2024-11-11T20:41:19,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742268_1444 (size=39749) 2024-11-11T20:41:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357739051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357739060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357739071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357739071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357739071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357739271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357739281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357739281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,292 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d107fc96e5344cac8d8540c07368bce6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d107fc96e5344cac8d8540c07368bce6 2024-11-11T20:41:19,297 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/B of fe28f5753443ca2f28d81a625707f258 into d107fc96e5344cac8d8540c07368bce6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:19,297 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:19,297 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/B, priority=13, startTime=1731357678838; duration=0sec 2024-11-11T20:41:19,297 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:19,297 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:B 2024-11-11T20:41:19,297 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:19,304 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:19,304 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/C is initiating minor compaction (all files) 2024-11-11T20:41:19,304 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/C in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:19,304 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4060a9c84814690979c82740cb603b3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/e36640d337e644668fa945f0464b65e0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4f88a94ee3248f8abde383d06b5f182] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=35.9 K 2024-11-11T20:41:19,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357739291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,304 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b4060a9c84814690979c82740cb603b3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731357675098 2024-11-11T20:41:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357739291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,305 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting e36640d337e644668fa945f0464b65e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731357675774 2024-11-11T20:41:19,306 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b4f88a94ee3248f8abde383d06b5f182, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731357677976 2024-11-11T20:41:19,314 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#C#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:19,314 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/a129a67ec3df4e01a43e3a94dc42eebc is 50, key is test_row_0/C:col10/1731357678643/Put/seqid=0 2024-11-11T20:41:19,316 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#A#compaction#373 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:19,317 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/1615175d4f5240cba675bd33beeb8289 is 175, key is test_row_0/A:col10/1731357678643/Put/seqid=0 2024-11-11T20:41:19,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742269_1445 (size=12595) 2024-11-11T20:41:19,358 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/a129a67ec3df4e01a43e3a94dc42eebc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a129a67ec3df4e01a43e3a94dc42eebc 2024-11-11T20:41:19,364 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/C of fe28f5753443ca2f28d81a625707f258 into a129a67ec3df4e01a43e3a94dc42eebc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:19,364 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:19,364 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/C, priority=13, startTime=1731357678838; duration=0sec 2024-11-11T20:41:19,364 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:19,364 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:C 2024-11-11T20:41:19,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742270_1446 (size=31549) 2024-11-11T20:41:19,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/415512790d5a499a9aaffae96fd8f87d 2024-11-11T20:41:19,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d6ff0fc5ce26489d87e56eed2c1a58c4 is 50, key is test_row_0/B:col10/1731357678753/Put/seqid=0 2024-11-11T20:41:19,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742271_1447 (size=12151) 2024-11-11T20:41:19,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d6ff0fc5ce26489d87e56eed2c1a58c4 2024-11-11T20:41:19,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/052f07f64e6149649de12dd50bbf04dd is 50, key is test_row_0/C:col10/1731357678753/Put/seqid=0 2024-11-11T20:41:19,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742272_1448 (size=12151) 2024-11-11T20:41:19,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/052f07f64e6149649de12dd50bbf04dd 2024-11-11T20:41:19,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/415512790d5a499a9aaffae96fd8f87d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/415512790d5a499a9aaffae96fd8f87d 2024-11-11T20:41:19,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/415512790d5a499a9aaffae96fd8f87d, entries=200, sequenceid=206, filesize=38.8 K 2024-11-11T20:41:19,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d6ff0fc5ce26489d87e56eed2c1a58c4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d6ff0fc5ce26489d87e56eed2c1a58c4 2024-11-11T20:41:19,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d6ff0fc5ce26489d87e56eed2c1a58c4, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T20:41:19,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/052f07f64e6149649de12dd50bbf04dd as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/052f07f64e6149649de12dd50bbf04dd 2024-11-11T20:41:19,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/052f07f64e6149649de12dd50bbf04dd, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T20:41:19,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for fe28f5753443ca2f28d81a625707f258 in 623ms, sequenceid=206, compaction requested=false 2024-11-11T20:41:19,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:19,587 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:19,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T20:41:19,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a3c9e71eaee345dcb230067c02e3d524_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357679586/Put/seqid=0 2024-11-11T20:41:19,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742273_1449 (size=14794) 2024-11-11T20:41:19,628 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:19,635 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a3c9e71eaee345dcb230067c02e3d524_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a3c9e71eaee345dcb230067c02e3d524_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:19,636 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/d039ff8a8a404a0e9f16604ab3ad26d0, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:19,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/d039ff8a8a404a0e9f16604ab3ad26d0 is 175, key is test_row_0/A:col10/1731357679586/Put/seqid=0 2024-11-11T20:41:19,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357739634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357739643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742274_1450 (size=39749) 2024-11-11T20:41:19,660 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/d039ff8a8a404a0e9f16604ab3ad26d0 2024-11-11T20:41:19,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357739645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357739646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357739646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/fadba4b9a3e34a73aa23f05e4333f993 is 50, key is test_row_0/B:col10/1731357679586/Put/seqid=0 2024-11-11T20:41:19,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742275_1451 (size=12151) 2024-11-11T20:41:19,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357739750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357739750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,778 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/1615175d4f5240cba675bd33beeb8289 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/1615175d4f5240cba675bd33beeb8289 2024-11-11T20:41:19,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357739771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357739771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357739771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,786 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/A of fe28f5753443ca2f28d81a625707f258 into 1615175d4f5240cba675bd33beeb8289(size=30.8 K), total size for store is 69.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:19,786 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:19,786 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/A, priority=13, startTime=1731357678838; duration=0sec 2024-11-11T20:41:19,786 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:19,786 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:A 2024-11-11T20:41:19,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357739971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357739971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:19,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357739991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:19,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357739991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357739991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/fadba4b9a3e34a73aa23f05e4333f993 2024-11-11T20:41:20,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/051f215163c843cdade370cfd5fb9baa is 50, key is test_row_0/C:col10/1731357679586/Put/seqid=0 2024-11-11T20:41:20,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742276_1452 (size=12151) 2024-11-11T20:41:20,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/051f215163c843cdade370cfd5fb9baa 2024-11-11T20:41:20,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/d039ff8a8a404a0e9f16604ab3ad26d0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d039ff8a8a404a0e9f16604ab3ad26d0 2024-11-11T20:41:20,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d039ff8a8a404a0e9f16604ab3ad26d0, entries=200, sequenceid=225, filesize=38.8 K 2024-11-11T20:41:20,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/fadba4b9a3e34a73aa23f05e4333f993 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/fadba4b9a3e34a73aa23f05e4333f993 2024-11-11T20:41:20,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/fadba4b9a3e34a73aa23f05e4333f993, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T20:41:20,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/051f215163c843cdade370cfd5fb9baa as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051f215163c843cdade370cfd5fb9baa 2024-11-11T20:41:20,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051f215163c843cdade370cfd5fb9baa, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T20:41:20,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for fe28f5753443ca2f28d81a625707f258 in 552ms, sequenceid=225, compaction requested=true 2024-11-11T20:41:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:20,139 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:20,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T20:41:20,139 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:20,140 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:20,140 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/A is initiating minor compaction (all files) 2024-11-11T20:41:20,140 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:20,140 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/A in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,140 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/B is initiating minor compaction (all files) 2024-11-11T20:41:20,140 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/B in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,140 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/1615175d4f5240cba675bd33beeb8289, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/415512790d5a499a9aaffae96fd8f87d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d039ff8a8a404a0e9f16604ab3ad26d0] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=108.4 K 2024-11-11T20:41:20,140 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,140 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d107fc96e5344cac8d8540c07368bce6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d6ff0fc5ce26489d87e56eed2c1a58c4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/fadba4b9a3e34a73aa23f05e4333f993] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=36.0 K 2024-11-11T20:41:20,141 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/1615175d4f5240cba675bd33beeb8289, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/415512790d5a499a9aaffae96fd8f87d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d039ff8a8a404a0e9f16604ab3ad26d0] 2024-11-11T20:41:20,141 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d107fc96e5344cac8d8540c07368bce6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731357677976 2024-11-11T20:41:20,141 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1615175d4f5240cba675bd33beeb8289, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731357677976 2024-11-11T20:41:20,141 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ff0fc5ce26489d87e56eed2c1a58c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731357678717 2024-11-11T20:41:20,141 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 415512790d5a499a9aaffae96fd8f87d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731357678717 2024-11-11T20:41:20,141 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fadba4b9a3e34a73aa23f05e4333f993, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731357678907 2024-11-11T20:41:20,142 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d039ff8a8a404a0e9f16604ab3ad26d0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731357678907 2024-11-11T20:41:20,158 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#B#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:20,158 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/815c0efcb0944093aec6ad2e88bfdd92 is 50, key is test_row_0/B:col10/1731357679586/Put/seqid=0 2024-11-11T20:41:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:41:20,181 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:20,181 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:20,181 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-11T20:41:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:41:20,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:20,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:20,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:20,191 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111a6434e03638740289cebd7ccb452d873_fe28f5753443ca2f28d81a625707f258 store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:20,193 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111a6434e03638740289cebd7ccb452d873_fe28f5753443ca2f28d81a625707f258, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:20,194 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a6434e03638740289cebd7ccb452d873_fe28f5753443ca2f28d81a625707f258 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:20,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742277_1453 (size=12697) 2024-11-11T20:41:20,240 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/815c0efcb0944093aec6ad2e88bfdd92 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/815c0efcb0944093aec6ad2e88bfdd92 2024-11-11T20:41:20,246 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/B of fe28f5753443ca2f28d81a625707f258 into 815c0efcb0944093aec6ad2e88bfdd92(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:20,246 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:20,246 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/B, priority=13, startTime=1731357680139; duration=0sec 2024-11-11T20:41:20,247 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:20,247 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:B 2024-11-11T20:41:20,247 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:20,249 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:20,249 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/C is initiating minor compaction (all files) 2024-11-11T20:41:20,249 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/C in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,249 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a129a67ec3df4e01a43e3a94dc42eebc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/052f07f64e6149649de12dd50bbf04dd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051f215163c843cdade370cfd5fb9baa] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=36.0 K 2024-11-11T20:41:20,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a129a67ec3df4e01a43e3a94dc42eebc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731357677976 2024-11-11T20:41:20,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 052f07f64e6149649de12dd50bbf04dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731357678717 2024-11-11T20:41:20,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 051f215163c843cdade370cfd5fb9baa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731357678907 2024-11-11T20:41:20,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742278_1454 (size=4469) 2024-11-11T20:41:20,266 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:20,266 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/54de92a15dda42098fff89432013ca2b is 50, key is test_row_0/C:col10/1731357679586/Put/seqid=0 2024-11-11T20:41:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:41:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:20,300 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:20,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T20:41:20,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742279_1455 (size=12697) 2024-11-11T20:41:20,326 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/54de92a15dda42098fff89432013ca2b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/54de92a15dda42098fff89432013ca2b 2024-11-11T20:41:20,331 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/C of fe28f5753443ca2f28d81a625707f258 into 54de92a15dda42098fff89432013ca2b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:20,331 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:20,331 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/C, priority=13, startTime=1731357680139; duration=0sec 2024-11-11T20:41:20,332 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:20,332 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:C 2024-11-11T20:41:20,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111a16150b62124e50bb69cec9f61a2ede_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357679642/Put/seqid=0 2024-11-11T20:41:20,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:41:20,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:20,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357740341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357740355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357740360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357740361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357740363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742280_1456 (size=17284) 2024-11-11T20:41:20,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357740471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357740480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:41:20,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:20,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:41:20,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357740490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357740491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357740491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:41:20,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:20,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,657 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#A#compaction#382 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:20,658 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/0749ebb9bed7413c9372cf47f6d06ff1 is 175, key is test_row_0/A:col10/1731357679586/Put/seqid=0 2024-11-11T20:41:20,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742281_1457 (size=31651) 2024-11-11T20:41:20,675 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/0749ebb9bed7413c9372cf47f6d06ff1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/0749ebb9bed7413c9372cf47f6d06ff1 2024-11-11T20:41:20,682 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/A of fe28f5753443ca2f28d81a625707f258 into 0749ebb9bed7413c9372cf47f6d06ff1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:20,682 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:20,682 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/A, priority=13, startTime=1731357680139; duration=0sec 2024-11-11T20:41:20,682 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:20,682 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:A 2024-11-11T20:41:20,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357740691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357740703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357740710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357740711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357740711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:20,787 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:20,793 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111a16150b62124e50bb69cec9f61a2ede_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111a16150b62124e50bb69cec9f61a2ede_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:20,794 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/80d1d6f8bf1e48598b6164b9c3790f29, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:20,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/80d1d6f8bf1e48598b6164b9c3790f29 is 175, key is test_row_0/A:col10/1731357679642/Put/seqid=0 2024-11-11T20:41:20,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:41:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:20,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:41:20,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742282_1458 (size=48389) 2024-11-11T20:41:20,835 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/80d1d6f8bf1e48598b6164b9c3790f29 2024-11-11T20:41:20,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/4a1a523fb3244c19ae91242cdf0a6ef3 is 50, key is test_row_0/B:col10/1731357679642/Put/seqid=0 2024-11-11T20:41:20,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742283_1459 (size=12151) 2024-11-11T20:41:20,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/4a1a523fb3244c19ae91242cdf0a6ef3 2024-11-11T20:41:20,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/b5b12dca2bce4c4b8b42a2804bd22841 is 50, key is test_row_0/C:col10/1731357679642/Put/seqid=0 2024-11-11T20:41:20,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742284_1460 (size=12151) 2024-11-11T20:41:20,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:41:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:20,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:21,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357741011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357741020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357741020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357741020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357741031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:41:21,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:21,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:21,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:21,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:21,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:21,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:41:21,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:21,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:21,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:21,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:21,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:21,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/b5b12dca2bce4c4b8b42a2804bd22841 2024-11-11T20:41:21,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/80d1d6f8bf1e48598b6164b9c3790f29 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/80d1d6f8bf1e48598b6164b9c3790f29 2024-11-11T20:41:21,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/80d1d6f8bf1e48598b6164b9c3790f29, entries=250, sequenceid=248, filesize=47.3 K 2024-11-11T20:41:21,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/4a1a523fb3244c19ae91242cdf0a6ef3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/4a1a523fb3244c19ae91242cdf0a6ef3 2024-11-11T20:41:21,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/4a1a523fb3244c19ae91242cdf0a6ef3, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T20:41:21,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/b5b12dca2bce4c4b8b42a2804bd22841 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b5b12dca2bce4c4b8b42a2804bd22841 2024-11-11T20:41:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:41:21,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b5b12dca2bce4c4b8b42a2804bd22841, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T20:41:21,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for fe28f5753443ca2f28d81a625707f258 in 1027ms, sequenceid=248, compaction requested=false 2024-11-11T20:41:21,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:21,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:41:21,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:21,421 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T20:41:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ffec2ff48890421d919cc5dcc05c07fb_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357680353/Put/seqid=0 2024-11-11T20:41:21,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742285_1461 (size=12454) 2024-11-11T20:41:21,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:21,456 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ffec2ff48890421d919cc5dcc05c07fb_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ffec2ff48890421d919cc5dcc05c07fb_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/45c2e6a5fe44428fb59af259066c40e3, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/45c2e6a5fe44428fb59af259066c40e3 is 175, key is test_row_0/A:col10/1731357680353/Put/seqid=0 2024-11-11T20:41:21,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742286_1462 (size=31255) 2024-11-11T20:41:21,477 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/45c2e6a5fe44428fb59af259066c40e3 2024-11-11T20:41:21,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d3315e29c27446bcbf871931fd50496a is 50, key is test_row_0/B:col10/1731357680353/Put/seqid=0 2024-11-11T20:41:21,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742287_1463 (size=12301) 2024-11-11T20:41:21,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:21,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357741565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357741566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357741567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357741575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357741576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357741681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357741681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357741681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357741683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357741700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357741890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357741901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357741902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357741902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,913 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d3315e29c27446bcbf871931fd50496a 2024-11-11T20:41:21,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/50043c2804ce41b0a110be4b7617fbd9 is 50, key is test_row_0/C:col10/1731357680353/Put/seqid=0 2024-11-11T20:41:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357741921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:21,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742288_1464 (size=12301) 2024-11-11T20:41:22,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357742211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357742221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357742221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357742221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357742231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,329 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/50043c2804ce41b0a110be4b7617fbd9 2024-11-11T20:41:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:41:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/45c2e6a5fe44428fb59af259066c40e3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/45c2e6a5fe44428fb59af259066c40e3 2024-11-11T20:41:22,344 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/45c2e6a5fe44428fb59af259066c40e3, entries=150, sequenceid=265, filesize=30.5 K 2024-11-11T20:41:22,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/d3315e29c27446bcbf871931fd50496a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3315e29c27446bcbf871931fd50496a 2024-11-11T20:41:22,355 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3315e29c27446bcbf871931fd50496a, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T20:41:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/50043c2804ce41b0a110be4b7617fbd9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/50043c2804ce41b0a110be4b7617fbd9 2024-11-11T20:41:22,362 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/50043c2804ce41b0a110be4b7617fbd9, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T20:41:22,364 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for fe28f5753443ca2f28d81a625707f258 in 943ms, sequenceid=265, compaction requested=true 2024-11-11T20:41:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-11T20:41:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-11T20:41:22,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T20:41:22,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-11-11T20:41:22,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.1890 sec 2024-11-11T20:41:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:22,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:22,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:41:22,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c099dfdb950d464a9acea3cd0ced3d83_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357681574/Put/seqid=0 2024-11-11T20:41:22,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742289_1465 (size=14994) 2024-11-11T20:41:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357742758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357742760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357742761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357742761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357742762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357742881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357742881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357742881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357742881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357742881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357743101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357743101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357743101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357743101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357743101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:23,152 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c099dfdb950d464a9acea3cd0ced3d83_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c099dfdb950d464a9acea3cd0ced3d83_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:23,153 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/654197f362b6475f9f0de65e19de31af, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:23,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/654197f362b6475f9f0de65e19de31af is 175, key is test_row_0/A:col10/1731357681574/Put/seqid=0 2024-11-11T20:41:23,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742290_1466 (size=39949) 2024-11-11T20:41:23,164 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/654197f362b6475f9f0de65e19de31af 2024-11-11T20:41:23,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/663710e7b5004c41842b66909edabf23 is 50, key is test_row_0/B:col10/1731357681574/Put/seqid=0 2024-11-11T20:41:23,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742291_1467 (size=12301) 2024-11-11T20:41:23,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/663710e7b5004c41842b66909edabf23 2024-11-11T20:41:23,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/d2f26fc62c29493fbcbd979ea9ea000a is 50, key is test_row_0/C:col10/1731357681574/Put/seqid=0 2024-11-11T20:41:23,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742292_1468 (size=12301) 2024-11-11T20:41:23,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/d2f26fc62c29493fbcbd979ea9ea000a 2024-11-11T20:41:23,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/654197f362b6475f9f0de65e19de31af as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/654197f362b6475f9f0de65e19de31af 2024-11-11T20:41:23,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/654197f362b6475f9f0de65e19de31af, entries=200, sequenceid=287, filesize=39.0 K 2024-11-11T20:41:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/663710e7b5004c41842b66909edabf23 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/663710e7b5004c41842b66909edabf23 2024-11-11T20:41:23,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/663710e7b5004c41842b66909edabf23, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T20:41:23,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/d2f26fc62c29493fbcbd979ea9ea000a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d2f26fc62c29493fbcbd979ea9ea000a 2024-11-11T20:41:23,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d2f26fc62c29493fbcbd979ea9ea000a, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T20:41:23,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for fe28f5753443ca2f28d81a625707f258 in 485ms, sequenceid=287, compaction requested=true 2024-11-11T20:41:23,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:23,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:23,223 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:23,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:23,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:23,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:23,223 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:23,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:23,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:23,224 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151244 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:23,224 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/A is initiating minor compaction (all files) 2024-11-11T20:41:23,224 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:23,224 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/B is initiating minor compaction (all files) 2024-11-11T20:41:23,224 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/A in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:23,224 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/B in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:23,224 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/0749ebb9bed7413c9372cf47f6d06ff1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/80d1d6f8bf1e48598b6164b9c3790f29, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/45c2e6a5fe44428fb59af259066c40e3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/654197f362b6475f9f0de65e19de31af] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=147.7 K 2024-11-11T20:41:23,224 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/815c0efcb0944093aec6ad2e88bfdd92, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/4a1a523fb3244c19ae91242cdf0a6ef3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3315e29c27446bcbf871931fd50496a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/663710e7b5004c41842b66909edabf23] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=48.3 K 2024-11-11T20:41:23,224 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:23,224 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/0749ebb9bed7413c9372cf47f6d06ff1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/80d1d6f8bf1e48598b6164b9c3790f29, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/45c2e6a5fe44428fb59af259066c40e3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/654197f362b6475f9f0de65e19de31af] 2024-11-11T20:41:23,224 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 815c0efcb0944093aec6ad2e88bfdd92, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731357678907 2024-11-11T20:41:23,224 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0749ebb9bed7413c9372cf47f6d06ff1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731357678907 2024-11-11T20:41:23,224 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a1a523fb3244c19ae91242cdf0a6ef3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731357679642 2024-11-11T20:41:23,224 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80d1d6f8bf1e48598b6164b9c3790f29, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731357679642 2024-11-11T20:41:23,225 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d3315e29c27446bcbf871931fd50496a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731357680353 2024-11-11T20:41:23,225 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c2e6a5fe44428fb59af259066c40e3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731357680353 2024-11-11T20:41:23,225 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 663710e7b5004c41842b66909edabf23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731357681564 2024-11-11T20:41:23,225 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654197f362b6475f9f0de65e19de31af, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731357681564 2024-11-11T20:41:23,240 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:23,242 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:23,242 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/be1d239085574bf485bac0420f6a21c9 is 50, key is test_row_0/B:col10/1731357681574/Put/seqid=0 2024-11-11T20:41:23,245 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411116748df59323441c7a9dbed7be7ac218d_fe28f5753443ca2f28d81a625707f258 store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:23,248 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411116748df59323441c7a9dbed7be7ac218d_fe28f5753443ca2f28d81a625707f258, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:23,248 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116748df59323441c7a9dbed7be7ac218d_fe28f5753443ca2f28d81a625707f258 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:23,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742293_1469 (size=12983) 2024-11-11T20:41:23,260 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/be1d239085574bf485bac0420f6a21c9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/be1d239085574bf485bac0420f6a21c9 2024-11-11T20:41:23,269 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe28f5753443ca2f28d81a625707f258/B of fe28f5753443ca2f28d81a625707f258 into be1d239085574bf485bac0420f6a21c9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:23,269 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:23,269 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/B, priority=12, startTime=1731357683223; duration=0sec 2024-11-11T20:41:23,269 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:23,269 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:B 2024-11-11T20:41:23,269 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:23,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742294_1470 (size=4469) 2024-11-11T20:41:23,273 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:23,273 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/C is initiating minor compaction (all files) 2024-11-11T20:41:23,273 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/C in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:23,273 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/54de92a15dda42098fff89432013ca2b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b5b12dca2bce4c4b8b42a2804bd22841, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/50043c2804ce41b0a110be4b7617fbd9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d2f26fc62c29493fbcbd979ea9ea000a] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=48.3 K 2024-11-11T20:41:23,274 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 54de92a15dda42098fff89432013ca2b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731357678907 2024-11-11T20:41:23,274 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b12dca2bce4c4b8b42a2804bd22841, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731357679642 2024-11-11T20:41:23,275 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 50043c2804ce41b0a110be4b7617fbd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731357680353 2024-11-11T20:41:23,277 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f26fc62c29493fbcbd979ea9ea000a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731357681564 2024-11-11T20:41:23,286 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#C#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:23,287 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/99de05a829174f9781844c6b0928606e is 50, key is test_row_0/C:col10/1731357681574/Put/seqid=0 2024-11-11T20:41:23,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742295_1471 (size=12983) 2024-11-11T20:41:23,310 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/99de05a829174f9781844c6b0928606e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/99de05a829174f9781844c6b0928606e 2024-11-11T20:41:23,316 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe28f5753443ca2f28d81a625707f258/C of fe28f5753443ca2f28d81a625707f258 into 99de05a829174f9781844c6b0928606e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:23,316 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:23,316 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/C, priority=12, startTime=1731357683223; duration=0sec 2024-11-11T20:41:23,316 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:23,316 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:C 2024-11-11T20:41:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:23,419 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:23,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T20:41:23,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117817fa0b008b478b83f4640c6d64de6f_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357682761/Put/seqid=0 2024-11-11T20:41:23,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742296_1472 (size=12454) 2024-11-11T20:41:23,457 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:23,472 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117817fa0b008b478b83f4640c6d64de6f_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117817fa0b008b478b83f4640c6d64de6f_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:23,473 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/22c768e277d047e3b818555eeda4c049, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:23,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/22c768e277d047e3b818555eeda4c049 is 175, key is test_row_0/A:col10/1731357682761/Put/seqid=0 2024-11-11T20:41:23,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357743457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742297_1473 (size=31255) 2024-11-11T20:41:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357743472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357743475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,493 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/22c768e277d047e3b818555eeda4c049 2024-11-11T20:41:23,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357743475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357743477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/dacad35a2a4748a2b435268f12cbf2fc is 50, key is test_row_0/B:col10/1731357682761/Put/seqid=0 2024-11-11T20:41:23,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742298_1474 (size=12301) 2024-11-11T20:41:23,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/dacad35a2a4748a2b435268f12cbf2fc 2024-11-11T20:41:23,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/92335b02d5ed4972b739b0dc2003fbbc is 50, key is test_row_0/C:col10/1731357682761/Put/seqid=0 2024-11-11T20:41:23,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742299_1475 (size=12301) 2024-11-11T20:41:23,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357743580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357743601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357743601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357743601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357743601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,670 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#A#compaction#394 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:23,671 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/6da47c28ba5b4fbf808a868c65ebc039 is 175, key is test_row_0/A:col10/1731357681574/Put/seqid=0 2024-11-11T20:41:23,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742300_1476 (size=31937) 2024-11-11T20:41:23,711 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/6da47c28ba5b4fbf808a868c65ebc039 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6da47c28ba5b4fbf808a868c65ebc039 2024-11-11T20:41:23,718 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe28f5753443ca2f28d81a625707f258/A of fe28f5753443ca2f28d81a625707f258 into 6da47c28ba5b4fbf808a868c65ebc039(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:23,718 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:23,718 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/A, priority=12, startTime=1731357683223; duration=0sec 2024-11-11T20:41:23,718 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:23,718 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:A 2024-11-11T20:41:23,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357743800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357743821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357743821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357743821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357743821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:23,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/92335b02d5ed4972b739b0dc2003fbbc 2024-11-11T20:41:23,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/22c768e277d047e3b818555eeda4c049 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/22c768e277d047e3b818555eeda4c049 2024-11-11T20:41:24,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/22c768e277d047e3b818555eeda4c049, entries=150, sequenceid=306, filesize=30.5 K 2024-11-11T20:41:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/dacad35a2a4748a2b435268f12cbf2fc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/dacad35a2a4748a2b435268f12cbf2fc 2024-11-11T20:41:24,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/dacad35a2a4748a2b435268f12cbf2fc, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T20:41:24,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/92335b02d5ed4972b739b0dc2003fbbc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/92335b02d5ed4972b739b0dc2003fbbc 2024-11-11T20:41:24,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/92335b02d5ed4972b739b0dc2003fbbc, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T20:41:24,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=120.76 KB/123660 for fe28f5753443ca2f28d81a625707f258 in 591ms, sequenceid=306, compaction requested=false 2024-11-11T20:41:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:24,116 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:24,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:41:24,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b192fe6cea424d04a3e76c5f8f12f5bc_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357684115/Put/seqid=0 2024-11-11T20:41:24,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742301_1477 (size=14994) 2024-11-11T20:41:24,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357744149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357744150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357744151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357744152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357744153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357744260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357744270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357744271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357744271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357744271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:41:24,341 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:24,341 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T20:41:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:41:24,344 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:24,344 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:24,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:41:24,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357744471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:41:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:24,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357744491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357744495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357744496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357744496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,527 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:24,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b192fe6cea424d04a3e76c5f8f12f5bc_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b192fe6cea424d04a3e76c5f8f12f5bc_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:24,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/b1c6ea7be90d40e8bbfb0f85fae0e409, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:24,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/b1c6ea7be90d40e8bbfb0f85fae0e409 is 175, key is test_row_0/A:col10/1731357684115/Put/seqid=0 2024-11-11T20:41:24,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742302_1478 (size=39949) 2024-11-11T20:41:24,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/b1c6ea7be90d40e8bbfb0f85fae0e409 2024-11-11T20:41:24,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/494fd7ac713144dbad6e5d4a081e3152 is 50, key is test_row_0/B:col10/1731357684115/Put/seqid=0 2024-11-11T20:41:24,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742303_1479 (size=12301) 2024-11-11T20:41:24,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:41:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:24,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:41:24,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357744781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:41:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:24,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357744810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357744810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357744810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357744810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:24,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:41:24,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:24,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:24,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:24,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/494fd7ac713144dbad6e5d4a081e3152 2024-11-11T20:41:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:24,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/bae9b0b4eaff49afbb18bb75c063c464 is 50, key is test_row_0/C:col10/1731357684115/Put/seqid=0 2024-11-11T20:41:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:41:24,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742304_1480 (size=12301) 2024-11-11T20:41:25,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:41:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:25,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:25,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:41:25,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:25,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:25,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357745300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:25,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357745320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357745320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357745321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:25,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357745331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:25,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/bae9b0b4eaff49afbb18bb75c063c464 2024-11-11T20:41:25,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/b1c6ea7be90d40e8bbfb0f85fae0e409 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/b1c6ea7be90d40e8bbfb0f85fae0e409 2024-11-11T20:41:25,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/b1c6ea7be90d40e8bbfb0f85fae0e409, entries=200, sequenceid=329, filesize=39.0 K 2024-11-11T20:41:25,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/494fd7ac713144dbad6e5d4a081e3152 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/494fd7ac713144dbad6e5d4a081e3152 2024-11-11T20:41:25,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/494fd7ac713144dbad6e5d4a081e3152, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T20:41:25,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/bae9b0b4eaff49afbb18bb75c063c464 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/bae9b0b4eaff49afbb18bb75c063c464 2024-11-11T20:41:25,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/bae9b0b4eaff49afbb18bb75c063c464, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T20:41:25,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for fe28f5753443ca2f28d81a625707f258 in 1273ms, sequenceid=329, compaction requested=true 2024-11-11T20:41:25,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:25,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:25,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:25,389 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:25,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:25,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:25,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:25,389 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:25,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:25,390 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:25,390 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:25,390 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/B is initiating minor compaction (all files) 2024-11-11T20:41:25,390 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/A is initiating minor compaction (all files) 2024-11-11T20:41:25,390 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/B in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:25,390 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/A in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:25,390 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/be1d239085574bf485bac0420f6a21c9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/dacad35a2a4748a2b435268f12cbf2fc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/494fd7ac713144dbad6e5d4a081e3152] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=36.7 K 2024-11-11T20:41:25,390 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6da47c28ba5b4fbf808a868c65ebc039, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/22c768e277d047e3b818555eeda4c049, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/b1c6ea7be90d40e8bbfb0f85fae0e409] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=100.7 K 2024-11-11T20:41:25,390 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:25,390 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6da47c28ba5b4fbf808a868c65ebc039, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/22c768e277d047e3b818555eeda4c049, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/b1c6ea7be90d40e8bbfb0f85fae0e409] 2024-11-11T20:41:25,390 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting be1d239085574bf485bac0420f6a21c9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731357681564 2024-11-11T20:41:25,391 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting dacad35a2a4748a2b435268f12cbf2fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731357682761 2024-11-11T20:41:25,391 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da47c28ba5b4fbf808a868c65ebc039, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731357681564 2024-11-11T20:41:25,391 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 494fd7ac713144dbad6e5d4a081e3152, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357683455 2024-11-11T20:41:25,391 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22c768e277d047e3b818555eeda4c049, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731357682761 2024-11-11T20:41:25,392 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c6ea7be90d40e8bbfb0f85fae0e409, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357683455 2024-11-11T20:41:25,406 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:25,407 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/9a65d1c134414538ae834e033968a509 is 50, key is test_row_0/B:col10/1731357684115/Put/seqid=0 2024-11-11T20:41:25,408 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:25,410 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111183387815de6f472986def5a5b67555bf_fe28f5753443ca2f28d81a625707f258 store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:25,412 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111183387815de6f472986def5a5b67555bf_fe28f5753443ca2f28d81a625707f258, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:25,412 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111183387815de6f472986def5a5b67555bf_fe28f5753443ca2f28d81a625707f258 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:25,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:41:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:25,416 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T20:41:25,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742305_1481 (size=13085) 2024-11-11T20:41:25,425 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/9a65d1c134414538ae834e033968a509 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/9a65d1c134414538ae834e033968a509 2024-11-11T20:41:25,437 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/B of fe28f5753443ca2f28d81a625707f258 into 9a65d1c134414538ae834e033968a509(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:25,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:25,437 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/B, priority=13, startTime=1731357685389; duration=0sec 2024-11-11T20:41:25,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:25,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:B 2024-11-11T20:41:25,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:25,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111174ab7a1370464e18a7f296e8774c0a9f_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357684151/Put/seqid=0 2024-11-11T20:41:25,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742306_1482 (size=4469) 2024-11-11T20:41:25,445 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:25,445 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/C is initiating minor compaction (all files) 2024-11-11T20:41:25,445 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/C in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:25,445 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/99de05a829174f9781844c6b0928606e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/92335b02d5ed4972b739b0dc2003fbbc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/bae9b0b4eaff49afbb18bb75c063c464] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=36.7 K 2024-11-11T20:41:25,446 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#A#compaction#403 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:25,447 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/cd1f5fdd8abd47cc94c6ab8fb910b0cc is 175, key is test_row_0/A:col10/1731357684115/Put/seqid=0 2024-11-11T20:41:25,453 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 99de05a829174f9781844c6b0928606e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731357681564 2024-11-11T20:41:25,453 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 92335b02d5ed4972b739b0dc2003fbbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731357682761 2024-11-11T20:41:25,454 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting bae9b0b4eaff49afbb18bb75c063c464, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357683455 2024-11-11T20:41:25,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742307_1483 (size=12454) 2024-11-11T20:41:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:25,469 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111174ab7a1370464e18a7f296e8774c0a9f_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111174ab7a1370464e18a7f296e8774c0a9f_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:25,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/3e2e447a1a5a44899825d44caecba01d, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/3e2e447a1a5a44899825d44caecba01d is 175, key is test_row_0/A:col10/1731357684151/Put/seqid=0 2024-11-11T20:41:25,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742308_1484 (size=32039) 2024-11-11T20:41:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:41:25,481 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#C#compaction#405 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:25,481 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/086573b42abd47c0aa6fb8e6a18ed23f is 50, key is test_row_0/C:col10/1731357684115/Put/seqid=0 2024-11-11T20:41:25,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742309_1485 (size=31255) 2024-11-11T20:41:25,484 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/cd1f5fdd8abd47cc94c6ab8fb910b0cc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/cd1f5fdd8abd47cc94c6ab8fb910b0cc 2024-11-11T20:41:25,489 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/A of fe28f5753443ca2f28d81a625707f258 into cd1f5fdd8abd47cc94c6ab8fb910b0cc(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:25,489 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:25,489 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/A, priority=13, startTime=1731357685389; duration=0sec 2024-11-11T20:41:25,489 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:25,489 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:A 2024-11-11T20:41:25,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742310_1486 (size=13085) 2024-11-11T20:41:25,884 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/3e2e447a1a5a44899825d44caecba01d 2024-11-11T20:41:25,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/3ab176f31b25453299436b9c60333c7d is 50, key is test_row_0/B:col10/1731357684151/Put/seqid=0 2024-11-11T20:41:25,910 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/086573b42abd47c0aa6fb8e6a18ed23f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/086573b42abd47c0aa6fb8e6a18ed23f 2024-11-11T20:41:25,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742311_1487 (size=12301) 2024-11-11T20:41:25,914 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/C of fe28f5753443ca2f28d81a625707f258 into 086573b42abd47c0aa6fb8e6a18ed23f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:25,914 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:25,914 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/C, priority=13, startTime=1731357685389; duration=0sec 2024-11-11T20:41:25,914 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:25,915 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:C 2024-11-11T20:41:26,310 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/3ab176f31b25453299436b9c60333c7d 2024-11-11T20:41:26,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/0ff7fa2839f24a03b4d2591c1e4fbc86 is 50, key is test_row_0/C:col10/1731357684151/Put/seqid=0 2024-11-11T20:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:26,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:26,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742312_1488 (size=12301) 2024-11-11T20:41:26,334 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/0ff7fa2839f24a03b4d2591c1e4fbc86 2024-11-11T20:41:26,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/3e2e447a1a5a44899825d44caecba01d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/3e2e447a1a5a44899825d44caecba01d 2024-11-11T20:41:26,343 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/3e2e447a1a5a44899825d44caecba01d, entries=150, sequenceid=343, filesize=30.5 K 2024-11-11T20:41:26,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/3ab176f31b25453299436b9c60333c7d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/3ab176f31b25453299436b9c60333c7d 2024-11-11T20:41:26,347 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/3ab176f31b25453299436b9c60333c7d, entries=150, sequenceid=343, filesize=12.0 K 2024-11-11T20:41:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/0ff7fa2839f24a03b4d2591c1e4fbc86 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0ff7fa2839f24a03b4d2591c1e4fbc86 2024-11-11T20:41:26,354 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0ff7fa2839f24a03b4d2591c1e4fbc86, entries=150, sequenceid=343, filesize=12.0 K 2024-11-11T20:41:26,354 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=60.38 KB/61830 for fe28f5753443ca2f28d81a625707f258 in 938ms, sequenceid=343, compaction requested=false 2024-11-11T20:41:26,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:26,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:26,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T20:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T20:41:26,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T20:41:26,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2024-11-11T20:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:26,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.0170 sec 2024-11-11T20:41:26,361 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:26,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T20:41:26,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115c576df9325041f2b849a3786805c2e1_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357686357/Put/seqid=0 2024-11-11T20:41:26,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742313_1489 (size=14994) 2024-11-11T20:41:26,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357746399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357746399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357746400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357746401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357746410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:41:26,491 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:26,491 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T20:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:41:26,493 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:26,493 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:26,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:26,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357746521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357746521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357746521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357746521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357746521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:41:26,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:26,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:26,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:26,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:26,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:26,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:26,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357746730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357746730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357746730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357746740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357746741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:26,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:26,794 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115c576df9325041f2b849a3786805c2e1_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115c576df9325041f2b849a3786805c2e1_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:26,794 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f02b483ae25548caa68e985a670d8ecc, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:26,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f02b483ae25548caa68e985a670d8ecc is 175, key is test_row_0/A:col10/1731357686357/Put/seqid=0 2024-11-11T20:41:26,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:26,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:26,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:26,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:26,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:26,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:26,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742314_1490 (size=39949) 2024-11-11T20:41:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:41:26,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:26,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:26,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357747041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357747050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357747050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357747051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357747051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:41:27,199 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f02b483ae25548caa68e985a670d8ecc 2024-11-11T20:41:27,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/f139303d28a5422399749c5a25627892 is 50, key is test_row_0/B:col10/1731357686357/Put/seqid=0 2024-11-11T20:41:27,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742315_1491 (size=12301) 2024-11-11T20:41:27,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:27,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:27,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:27,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44686 deadline: 1731357747551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:27,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:27,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44724 deadline: 1731357747571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44740 deadline: 1731357747571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1731357747571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44696 deadline: 1731357747571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:41:27,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/f139303d28a5422399749c5a25627892 2024-11-11T20:41:27,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/051c4c5ef74c41d7b5f88e467bd8f279 is 50, key is test_row_0/C:col10/1731357686357/Put/seqid=0 2024-11-11T20:41:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742316_1492 (size=12301) 2024-11-11T20:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:41:27,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:27,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:27,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:27,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:27,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:27,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:28,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:28,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:28,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:28,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:28,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:28,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:28,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/051c4c5ef74c41d7b5f88e467bd8f279 2024-11-11T20:41:28,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/f02b483ae25548caa68e985a670d8ecc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f02b483ae25548caa68e985a670d8ecc 2024-11-11T20:41:28,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f02b483ae25548caa68e985a670d8ecc, entries=200, sequenceid=362, filesize=39.0 K 2024-11-11T20:41:28,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/f139303d28a5422399749c5a25627892 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f139303d28a5422399749c5a25627892 2024-11-11T20:41:28,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f139303d28a5422399749c5a25627892, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T20:41:28,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/051c4c5ef74c41d7b5f88e467bd8f279 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051c4c5ef74c41d7b5f88e467bd8f279 2024-11-11T20:41:28,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051c4c5ef74c41d7b5f88e467bd8f279, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T20:41:28,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for fe28f5753443ca2f28d81a625707f258 in 1686ms, sequenceid=362, compaction requested=true 2024-11-11T20:41:28,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:28,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:28,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:28,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:28,047 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:28,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:28,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe28f5753443ca2f28d81a625707f258:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:28,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T20:41:28,048 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:28,048 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:28,048 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/A is initiating minor compaction (all files) 2024-11-11T20:41:28,048 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/A in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:28,049 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/cd1f5fdd8abd47cc94c6ab8fb910b0cc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/3e2e447a1a5a44899825d44caecba01d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f02b483ae25548caa68e985a670d8ecc] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=100.8 K 2024-11-11T20:41:28,049 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:28,049 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/cd1f5fdd8abd47cc94c6ab8fb910b0cc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/3e2e447a1a5a44899825d44caecba01d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f02b483ae25548caa68e985a670d8ecc] 2024-11-11T20:41:28,049 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:28,049 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/B is initiating minor compaction (all files) 2024-11-11T20:41:28,049 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/B in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:28,049 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/9a65d1c134414538ae834e033968a509, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/3ab176f31b25453299436b9c60333c7d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f139303d28a5422399749c5a25627892] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=36.8 K 2024-11-11T20:41:28,050 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd1f5fdd8abd47cc94c6ab8fb910b0cc, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357683455 2024-11-11T20:41:28,050 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a65d1c134414538ae834e033968a509, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357683455 2024-11-11T20:41:28,050 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ab176f31b25453299436b9c60333c7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731357684150 2024-11-11T20:41:28,050 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2e447a1a5a44899825d44caecba01d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731357684150 2024-11-11T20:41:28,050 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f139303d28a5422399749c5a25627892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731357686357 2024-11-11T20:41:28,050 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f02b483ae25548caa68e985a670d8ecc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731357686348 2024-11-11T20:41:28,055 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:28,057 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#B#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:28,057 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/206bee059109481aa245b9cb0943df28 is 50, key is test_row_0/B:col10/1731357686357/Put/seqid=0 2024-11-11T20:41:28,066 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111f9a93c32d2a24d06958f381eda7658cc_fe28f5753443ca2f28d81a625707f258 store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:28,068 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111f9a93c32d2a24d06958f381eda7658cc_fe28f5753443ca2f28d81a625707f258, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:28,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742317_1493 (size=13187) 2024-11-11T20:41:28,068 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f9a93c32d2a24d06958f381eda7658cc_fe28f5753443ca2f28d81a625707f258 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:28,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742318_1494 (size=4469) 2024-11-11T20:41:28,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:41:28,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:28,174 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T20:41:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111192ab81587a4a4f84aa528675f5eb2916_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357686409/Put/seqid=0 2024-11-11T20:41:28,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742319_1495 (size=12454) 2024-11-11T20:41:28,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:28,208 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111192ab81587a4a4f84aa528675f5eb2916_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111192ab81587a4a4f84aa528675f5eb2916_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:28,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/4dd5c07b84b2431e91569a23f22da41c, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:28,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/4dd5c07b84b2431e91569a23f22da41c is 175, key is test_row_0/A:col10/1731357686409/Put/seqid=0 2024-11-11T20:41:28,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742320_1496 (size=31255) 2024-11-11T20:41:28,421 INFO [Thread-1756 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1756. 2024-11-11T20:41:28,421 DEBUG [Thread-1756 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,421 DEBUG [Thread-1756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,421 DEBUG [Thread-1756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,425 INFO [Thread-1750 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1750. 2024-11-11T20:41:28,425 DEBUG [Thread-1750 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,425 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,426 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,426 INFO [Thread-1754 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1754. 2024-11-11T20:41:28,426 DEBUG [Thread-1754 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,426 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,426 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,428 INFO [Thread-1752 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1752. 2024-11-11T20:41:28,428 DEBUG [Thread-1752 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,428 DEBUG [Thread-1752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,428 DEBUG [Thread-1752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,432 INFO [Thread-1758 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1758. 2024-11-11T20:41:28,432 DEBUG [Thread-1758 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,432 DEBUG [Thread-1758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,432 DEBUG [Thread-1758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,474 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/206bee059109481aa245b9cb0943df28 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/206bee059109481aa245b9cb0943df28 2024-11-11T20:41:28,479 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/B of fe28f5753443ca2f28d81a625707f258 into 206bee059109481aa245b9cb0943df28(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:28,479 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#A#compaction#411 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:28,479 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:28,479 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/B, priority=13, startTime=1731357688047; duration=0sec 2024-11-11T20:41:28,479 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:28,479 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:B 2024-11-11T20:41:28,479 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:28,479 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/42a7ddc9fe464923a56d515ef683a997 is 175, key is test_row_0/A:col10/1731357686357/Put/seqid=0 2024-11-11T20:41:28,480 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:28,480 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe28f5753443ca2f28d81a625707f258/C is initiating minor compaction (all files) 2024-11-11T20:41:28,480 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe28f5753443ca2f28d81a625707f258/C in TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:28,480 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/086573b42abd47c0aa6fb8e6a18ed23f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0ff7fa2839f24a03b4d2591c1e4fbc86, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051c4c5ef74c41d7b5f88e467bd8f279] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp, totalSize=36.8 K 2024-11-11T20:41:28,481 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 086573b42abd47c0aa6fb8e6a18ed23f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731357683455 2024-11-11T20:41:28,481 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff7fa2839f24a03b4d2591c1e4fbc86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731357684150 2024-11-11T20:41:28,481 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 051c4c5ef74c41d7b5f88e467bd8f279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731357686357 2024-11-11T20:41:28,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742321_1497 (size=32141) 2024-11-11T20:41:28,486 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/42a7ddc9fe464923a56d515ef683a997 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/42a7ddc9fe464923a56d515ef683a997 2024-11-11T20:41:28,487 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe28f5753443ca2f28d81a625707f258#C#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:28,487 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/93dec11c7fe242eb8e563fc26e3377dd is 50, key is test_row_0/C:col10/1731357686357/Put/seqid=0 2024-11-11T20:41:28,489 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/A of fe28f5753443ca2f28d81a625707f258 into 42a7ddc9fe464923a56d515ef683a997(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:28,489 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:28,489 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/A, priority=13, startTime=1731357688047; duration=0sec 2024-11-11T20:41:28,490 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:28,490 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:A 2024-11-11T20:41:28,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742322_1498 (size=13187) 2024-11-11T20:41:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:28,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. as already flushing 2024-11-11T20:41:28,563 INFO [Thread-1739 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1739. 2024-11-11T20:41:28,563 DEBUG [Thread-1739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,563 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,563 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,582 INFO [Thread-1743 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1743. 2024-11-11T20:41:28,582 DEBUG [Thread-1743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,582 INFO [Thread-1741 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1741. 2024-11-11T20:41:28,582 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,582 DEBUG [Thread-1741 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,582 DEBUG [Thread-1741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,582 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,582 DEBUG [Thread-1741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,592 INFO [Thread-1745 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1745. 2024-11-11T20:41:28,592 DEBUG [Thread-1745 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,592 INFO [Thread-1747 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1747. 2024-11-11T20:41:28,592 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,592 DEBUG [Thread-1747 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:28,592 DEBUG [Thread-1747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,592 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,592 DEBUG [Thread-1747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:28,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:28,624 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/4dd5c07b84b2431e91569a23f22da41c 2024-11-11T20:41:28,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/0cb72f132967438985f46de6e981610e is 50, key is test_row_0/B:col10/1731357686409/Put/seqid=0 2024-11-11T20:41:28,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742323_1499 (size=12301) 2024-11-11T20:41:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:41:28,894 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/93dec11c7fe242eb8e563fc26e3377dd as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/93dec11c7fe242eb8e563fc26e3377dd 2024-11-11T20:41:28,898 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe28f5753443ca2f28d81a625707f258/C of fe28f5753443ca2f28d81a625707f258 into 93dec11c7fe242eb8e563fc26e3377dd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:28,898 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:28,898 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., storeName=fe28f5753443ca2f28d81a625707f258/C, priority=13, startTime=1731357688047; duration=0sec 2024-11-11T20:41:28,898 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:28,898 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe28f5753443ca2f28d81a625707f258:C 2024-11-11T20:41:29,033 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/0cb72f132967438985f46de6e981610e 2024-11-11T20:41:29,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/6445b01c2eb84e00a6487fdbc3e4351f is 50, key is test_row_0/C:col10/1731357686409/Put/seqid=0 2024-11-11T20:41:29,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742324_1500 (size=12301) 2024-11-11T20:41:29,443 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/6445b01c2eb84e00a6487fdbc3e4351f 2024-11-11T20:41:29,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/4dd5c07b84b2431e91569a23f22da41c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4dd5c07b84b2431e91569a23f22da41c 2024-11-11T20:41:29,450 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4dd5c07b84b2431e91569a23f22da41c, entries=150, sequenceid=382, filesize=30.5 K 2024-11-11T20:41:29,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/0cb72f132967438985f46de6e981610e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/0cb72f132967438985f46de6e981610e 2024-11-11T20:41:29,454 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/0cb72f132967438985f46de6e981610e, entries=150, sequenceid=382, filesize=12.0 K 2024-11-11T20:41:29,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/6445b01c2eb84e00a6487fdbc3e4351f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/6445b01c2eb84e00a6487fdbc3e4351f 2024-11-11T20:41:29,457 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/6445b01c2eb84e00a6487fdbc3e4351f, entries=150, sequenceid=382, filesize=12.0 K 2024-11-11T20:41:29,458 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=33.54 KB/34350 for fe28f5753443ca2f28d81a625707f258 in 1284ms, sequenceid=382, compaction requested=false 2024-11-11T20:41:29,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for fe28f5753443ca2f28d81a625707f258: 2024-11-11T20:41:29,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:29,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T20:41:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T20:41:29,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T20:41:29,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9660 sec 2024-11-11T20:41:29,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.9700 sec 2024-11-11T20:41:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:41:30,661 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1242 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3722 rows 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1255 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3765 rows 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1257 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3764 rows 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1254 2024-11-11T20:41:30,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3758 rows 2024-11-11T20:41:30,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1257 2024-11-11T20:41:30,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3767 rows 2024-11-11T20:41:30,662 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:41:30,662 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:41:30,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:30,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:30,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:30,663 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:41:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:41:30,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357690666"}]},"ts":"1731357690666"} 2024-11-11T20:41:30,667 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:41:30,667 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:41:30,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:41:30,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, UNASSIGN}] 2024-11-11T20:41:30,670 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, UNASSIGN 2024-11-11T20:41:30,670 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=fe28f5753443ca2f28d81a625707f258, regionState=CLOSING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:30,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, UNASSIGN because future has completed 2024-11-11T20:41:30,671 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:41:30,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:41:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:41:30,823 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:30,823 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:41:30,824 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing fe28f5753443ca2f28d81a625707f258, disabling compactions & flushes 2024-11-11T20:41:30,824 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:30,824 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:30,824 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. after waiting 0 ms 2024-11-11T20:41:30,824 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:30,824 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing fe28f5753443ca2f28d81a625707f258 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T20:41:30,829 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111138584baddcec48d7ac041bfff9668a86_fe28f5753443ca2f28d81a625707f258 is 50, key is test_row_0/A:col10/1731357688591/Put/seqid=0 2024-11-11T20:41:30,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742325_1501 (size=12454) 2024-11-11T20:41:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:41:31,232 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:31,236 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111138584baddcec48d7ac041bfff9668a86_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111138584baddcec48d7ac041bfff9668a86_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:31,237 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/e8a771f3ffe140e4b614d6bc9e729ad9, store: [table=TestAcidGuarantees family=A region=fe28f5753443ca2f28d81a625707f258] 2024-11-11T20:41:31,237 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/e8a771f3ffe140e4b614d6bc9e729ad9 is 175, key is test_row_0/A:col10/1731357688591/Put/seqid=0 2024-11-11T20:41:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742326_1502 (size=31255) 2024-11-11T20:41:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:41:31,641 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/e8a771f3ffe140e4b614d6bc9e729ad9 2024-11-11T20:41:31,646 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/1851673874fc49acb0b7033aeefe8a24 is 50, key is test_row_0/B:col10/1731357688591/Put/seqid=0 2024-11-11T20:41:31,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742327_1503 (size=12301) 2024-11-11T20:41:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:41:32,051 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/1851673874fc49acb0b7033aeefe8a24 2024-11-11T20:41:32,062 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/f211c2343e0c400089eeae66abce7d66 is 50, key is test_row_0/C:col10/1731357688591/Put/seqid=0 2024-11-11T20:41:32,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742328_1504 (size=12301) 2024-11-11T20:41:32,467 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/f211c2343e0c400089eeae66abce7d66 2024-11-11T20:41:32,473 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/A/e8a771f3ffe140e4b614d6bc9e729ad9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/e8a771f3ffe140e4b614d6bc9e729ad9 2024-11-11T20:41:32,476 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/e8a771f3ffe140e4b614d6bc9e729ad9, entries=150, sequenceid=393, filesize=30.5 K 2024-11-11T20:41:32,477 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/B/1851673874fc49acb0b7033aeefe8a24 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/1851673874fc49acb0b7033aeefe8a24 2024-11-11T20:41:32,481 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/1851673874fc49acb0b7033aeefe8a24, entries=150, sequenceid=393, filesize=12.0 K 2024-11-11T20:41:32,482 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/.tmp/C/f211c2343e0c400089eeae66abce7d66 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/f211c2343e0c400089eeae66abce7d66 2024-11-11T20:41:32,485 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/f211c2343e0c400089eeae66abce7d66, entries=150, sequenceid=393, filesize=12.0 K 2024-11-11T20:41:32,486 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fe28f5753443ca2f28d81a625707f258 in 1662ms, sequenceid=393, compaction requested=true 2024-11-11T20:41:32,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a1c7a27c061c4e43bacda7d05860e784, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4747bf01bc134e3fbb8de0dad7f54328, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/96cfa0d12995484b8c32152119c916ac, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6e3b4cdee5474d659bd752154a103378, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d968f98a8b194864a4f853565b31f68b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/239839805fa2407e93c6fc4fd16125c6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f47e8bb5b08644c088017a7145a406c2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f89c7c54cc9046d89f6f940d867b0aaf, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/5343fffdfef148b3a4a6811d0f8f9688, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/8ab06d608cd647cc9a49144a7095d38c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a4ba48a9f7b14ea3a6d87df7499c51a6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/dc1b250f7d744c6886a634fa7c777d56, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/1615175d4f5240cba675bd33beeb8289, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/7bba27a67d8a463b9c9739ea35712eb5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/415512790d5a499a9aaffae96fd8f87d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d039ff8a8a404a0e9f16604ab3ad26d0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/0749ebb9bed7413c9372cf47f6d06ff1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/80d1d6f8bf1e48598b6164b9c3790f29, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/45c2e6a5fe44428fb59af259066c40e3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/654197f362b6475f9f0de65e19de31af, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6da47c28ba5b4fbf808a868c65ebc039, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/22c768e277d047e3b818555eeda4c049, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/b1c6ea7be90d40e8bbfb0f85fae0e409, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/cd1f5fdd8abd47cc94c6ab8fb910b0cc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/3e2e447a1a5a44899825d44caecba01d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f02b483ae25548caa68e985a670d8ecc] to archive 2024-11-11T20:41:32,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:41:32,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a1c7a27c061c4e43bacda7d05860e784 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a1c7a27c061c4e43bacda7d05860e784 2024-11-11T20:41:32,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4747bf01bc134e3fbb8de0dad7f54328 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4747bf01bc134e3fbb8de0dad7f54328 2024-11-11T20:41:32,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/96cfa0d12995484b8c32152119c916ac to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/96cfa0d12995484b8c32152119c916ac 2024-11-11T20:41:32,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6e3b4cdee5474d659bd752154a103378 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6e3b4cdee5474d659bd752154a103378 2024-11-11T20:41:32,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d968f98a8b194864a4f853565b31f68b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d968f98a8b194864a4f853565b31f68b 2024-11-11T20:41:32,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/239839805fa2407e93c6fc4fd16125c6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/239839805fa2407e93c6fc4fd16125c6 2024-11-11T20:41:32,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f47e8bb5b08644c088017a7145a406c2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f47e8bb5b08644c088017a7145a406c2 2024-11-11T20:41:32,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f89c7c54cc9046d89f6f940d867b0aaf to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f89c7c54cc9046d89f6f940d867b0aaf 2024-11-11T20:41:32,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/5343fffdfef148b3a4a6811d0f8f9688 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/5343fffdfef148b3a4a6811d0f8f9688 2024-11-11T20:41:32,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/8ab06d608cd647cc9a49144a7095d38c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/8ab06d608cd647cc9a49144a7095d38c 2024-11-11T20:41:32,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a4ba48a9f7b14ea3a6d87df7499c51a6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/a4ba48a9f7b14ea3a6d87df7499c51a6 2024-11-11T20:41:32,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/dc1b250f7d744c6886a634fa7c777d56 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/dc1b250f7d744c6886a634fa7c777d56 2024-11-11T20:41:32,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/1615175d4f5240cba675bd33beeb8289 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/1615175d4f5240cba675bd33beeb8289 2024-11-11T20:41:32,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/7bba27a67d8a463b9c9739ea35712eb5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/7bba27a67d8a463b9c9739ea35712eb5 2024-11-11T20:41:32,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/415512790d5a499a9aaffae96fd8f87d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/415512790d5a499a9aaffae96fd8f87d 2024-11-11T20:41:32,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d039ff8a8a404a0e9f16604ab3ad26d0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/d039ff8a8a404a0e9f16604ab3ad26d0 2024-11-11T20:41:32,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/0749ebb9bed7413c9372cf47f6d06ff1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/0749ebb9bed7413c9372cf47f6d06ff1 2024-11-11T20:41:32,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/80d1d6f8bf1e48598b6164b9c3790f29 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/80d1d6f8bf1e48598b6164b9c3790f29 2024-11-11T20:41:32,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/45c2e6a5fe44428fb59af259066c40e3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/45c2e6a5fe44428fb59af259066c40e3 2024-11-11T20:41:32,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/654197f362b6475f9f0de65e19de31af to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/654197f362b6475f9f0de65e19de31af 2024-11-11T20:41:32,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6da47c28ba5b4fbf808a868c65ebc039 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/6da47c28ba5b4fbf808a868c65ebc039 2024-11-11T20:41:32,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/22c768e277d047e3b818555eeda4c049 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/22c768e277d047e3b818555eeda4c049 2024-11-11T20:41:32,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/b1c6ea7be90d40e8bbfb0f85fae0e409 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/b1c6ea7be90d40e8bbfb0f85fae0e409 2024-11-11T20:41:32,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/cd1f5fdd8abd47cc94c6ab8fb910b0cc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/cd1f5fdd8abd47cc94c6ab8fb910b0cc 2024-11-11T20:41:32,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/3e2e447a1a5a44899825d44caecba01d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/3e2e447a1a5a44899825d44caecba01d 2024-11-11T20:41:32,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f02b483ae25548caa68e985a670d8ecc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/f02b483ae25548caa68e985a670d8ecc 2024-11-11T20:41:32,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/490205f52d6046aa8c1cbaa1722d58fd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/608385766ffe49739f68edcf82596f69, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ff6686d7f1574ba8ba63afd3bed77c49, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a2cbc01393024b2cb3ff86c93058b95d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/b26aeb1f16154492bb21bf7dc4ca16e2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a90f2aa0a60b4bf89d5045ce39a79f5c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/1afd0743359f4056986de7de0029237b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ac6b2af92f7b45889a7cab7795e06c9b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/957b2be6d48c40de8a57b5eb5856963e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ccb2d4292fd84f5a9e2c6fbbb684f9b1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f583e4ae41ca491aa0e42b12fe76d906, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3d31050028144bea5dd467ced52f101, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d107fc96e5344cac8d8540c07368bce6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ee1cf7e02bbf4dccbbb11ca4a7afa102, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d6ff0fc5ce26489d87e56eed2c1a58c4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/815c0efcb0944093aec6ad2e88bfdd92, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/fadba4b9a3e34a73aa23f05e4333f993, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/4a1a523fb3244c19ae91242cdf0a6ef3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3315e29c27446bcbf871931fd50496a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/be1d239085574bf485bac0420f6a21c9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/663710e7b5004c41842b66909edabf23, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/dacad35a2a4748a2b435268f12cbf2fc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/9a65d1c134414538ae834e033968a509, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/494fd7ac713144dbad6e5d4a081e3152, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/3ab176f31b25453299436b9c60333c7d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f139303d28a5422399749c5a25627892] to archive 2024-11-11T20:41:32,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:41:32,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/490205f52d6046aa8c1cbaa1722d58fd to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/490205f52d6046aa8c1cbaa1722d58fd 2024-11-11T20:41:32,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/608385766ffe49739f68edcf82596f69 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/608385766ffe49739f68edcf82596f69 2024-11-11T20:41:32,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ff6686d7f1574ba8ba63afd3bed77c49 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ff6686d7f1574ba8ba63afd3bed77c49 2024-11-11T20:41:32,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a2cbc01393024b2cb3ff86c93058b95d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a2cbc01393024b2cb3ff86c93058b95d 2024-11-11T20:41:32,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/b26aeb1f16154492bb21bf7dc4ca16e2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/b26aeb1f16154492bb21bf7dc4ca16e2 2024-11-11T20:41:32,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a90f2aa0a60b4bf89d5045ce39a79f5c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/a90f2aa0a60b4bf89d5045ce39a79f5c 2024-11-11T20:41:32,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/1afd0743359f4056986de7de0029237b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/1afd0743359f4056986de7de0029237b 2024-11-11T20:41:32,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ac6b2af92f7b45889a7cab7795e06c9b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ac6b2af92f7b45889a7cab7795e06c9b 2024-11-11T20:41:32,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/957b2be6d48c40de8a57b5eb5856963e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/957b2be6d48c40de8a57b5eb5856963e 2024-11-11T20:41:32,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ccb2d4292fd84f5a9e2c6fbbb684f9b1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ccb2d4292fd84f5a9e2c6fbbb684f9b1 2024-11-11T20:41:32,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f583e4ae41ca491aa0e42b12fe76d906 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f583e4ae41ca491aa0e42b12fe76d906 2024-11-11T20:41:32,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3d31050028144bea5dd467ced52f101 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3d31050028144bea5dd467ced52f101 2024-11-11T20:41:32,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d107fc96e5344cac8d8540c07368bce6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d107fc96e5344cac8d8540c07368bce6 2024-11-11T20:41:32,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ee1cf7e02bbf4dccbbb11ca4a7afa102 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/ee1cf7e02bbf4dccbbb11ca4a7afa102 2024-11-11T20:41:32,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d6ff0fc5ce26489d87e56eed2c1a58c4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d6ff0fc5ce26489d87e56eed2c1a58c4 2024-11-11T20:41:32,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/815c0efcb0944093aec6ad2e88bfdd92 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/815c0efcb0944093aec6ad2e88bfdd92 2024-11-11T20:41:32,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/fadba4b9a3e34a73aa23f05e4333f993 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/fadba4b9a3e34a73aa23f05e4333f993 2024-11-11T20:41:32,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/4a1a523fb3244c19ae91242cdf0a6ef3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/4a1a523fb3244c19ae91242cdf0a6ef3 2024-11-11T20:41:32,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3315e29c27446bcbf871931fd50496a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/d3315e29c27446bcbf871931fd50496a 2024-11-11T20:41:32,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/be1d239085574bf485bac0420f6a21c9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/be1d239085574bf485bac0420f6a21c9 2024-11-11T20:41:32,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/663710e7b5004c41842b66909edabf23 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/663710e7b5004c41842b66909edabf23 2024-11-11T20:41:32,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/dacad35a2a4748a2b435268f12cbf2fc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/dacad35a2a4748a2b435268f12cbf2fc 2024-11-11T20:41:32,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/9a65d1c134414538ae834e033968a509 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/9a65d1c134414538ae834e033968a509 2024-11-11T20:41:32,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/494fd7ac713144dbad6e5d4a081e3152 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/494fd7ac713144dbad6e5d4a081e3152 2024-11-11T20:41:32,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/3ab176f31b25453299436b9c60333c7d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/3ab176f31b25453299436b9c60333c7d 2024-11-11T20:41:32,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f139303d28a5422399749c5a25627892 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/f139303d28a5422399749c5a25627892 2024-11-11T20:41:32,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/2846e2cd878f4ea5b4f75c38a38993f3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/84f0fcca643043f4a3ee161541763a57, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0d3f034f608f41e2969399b2931db8da, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d02d5fc66acb4890b19dcf47b1924678, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/23d13ae3cd3a4f0a90d38b1a4ed05bb1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a62ec120d491486e8bd05e3563519706, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/787c6219251a4c82b636a686748b4c18, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d7c06f6f092444a8b173c21c17bb2297, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/4e598fdef2844637a1d4875d07bfba92, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4060a9c84814690979c82740cb603b3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/ab553dad304749a0aca2b79f81dfeb0c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/e36640d337e644668fa945f0464b65e0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a129a67ec3df4e01a43e3a94dc42eebc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4f88a94ee3248f8abde383d06b5f182, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/052f07f64e6149649de12dd50bbf04dd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/54de92a15dda42098fff89432013ca2b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051f215163c843cdade370cfd5fb9baa, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b5b12dca2bce4c4b8b42a2804bd22841, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/50043c2804ce41b0a110be4b7617fbd9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/99de05a829174f9781844c6b0928606e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d2f26fc62c29493fbcbd979ea9ea000a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/92335b02d5ed4972b739b0dc2003fbbc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/086573b42abd47c0aa6fb8e6a18ed23f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/bae9b0b4eaff49afbb18bb75c063c464, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0ff7fa2839f24a03b4d2591c1e4fbc86, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051c4c5ef74c41d7b5f88e467bd8f279] to archive 2024-11-11T20:41:32,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:41:32,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/2846e2cd878f4ea5b4f75c38a38993f3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/2846e2cd878f4ea5b4f75c38a38993f3 2024-11-11T20:41:32,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/84f0fcca643043f4a3ee161541763a57 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/84f0fcca643043f4a3ee161541763a57 2024-11-11T20:41:32,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0d3f034f608f41e2969399b2931db8da to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0d3f034f608f41e2969399b2931db8da 2024-11-11T20:41:32,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d02d5fc66acb4890b19dcf47b1924678 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d02d5fc66acb4890b19dcf47b1924678 2024-11-11T20:41:32,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/23d13ae3cd3a4f0a90d38b1a4ed05bb1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/23d13ae3cd3a4f0a90d38b1a4ed05bb1 2024-11-11T20:41:32,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a62ec120d491486e8bd05e3563519706 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a62ec120d491486e8bd05e3563519706 2024-11-11T20:41:32,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/787c6219251a4c82b636a686748b4c18 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/787c6219251a4c82b636a686748b4c18 2024-11-11T20:41:32,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d7c06f6f092444a8b173c21c17bb2297 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d7c06f6f092444a8b173c21c17bb2297 2024-11-11T20:41:32,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/4e598fdef2844637a1d4875d07bfba92 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/4e598fdef2844637a1d4875d07bfba92 2024-11-11T20:41:32,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4060a9c84814690979c82740cb603b3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4060a9c84814690979c82740cb603b3 2024-11-11T20:41:32,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/ab553dad304749a0aca2b79f81dfeb0c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/ab553dad304749a0aca2b79f81dfeb0c 2024-11-11T20:41:32,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/e36640d337e644668fa945f0464b65e0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/e36640d337e644668fa945f0464b65e0 2024-11-11T20:41:32,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a129a67ec3df4e01a43e3a94dc42eebc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/a129a67ec3df4e01a43e3a94dc42eebc 2024-11-11T20:41:32,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4f88a94ee3248f8abde383d06b5f182 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b4f88a94ee3248f8abde383d06b5f182 2024-11-11T20:41:32,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/052f07f64e6149649de12dd50bbf04dd to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/052f07f64e6149649de12dd50bbf04dd 2024-11-11T20:41:32,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/54de92a15dda42098fff89432013ca2b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/54de92a15dda42098fff89432013ca2b 2024-11-11T20:41:32,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051f215163c843cdade370cfd5fb9baa to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051f215163c843cdade370cfd5fb9baa 2024-11-11T20:41:32,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b5b12dca2bce4c4b8b42a2804bd22841 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/b5b12dca2bce4c4b8b42a2804bd22841 2024-11-11T20:41:32,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/50043c2804ce41b0a110be4b7617fbd9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/50043c2804ce41b0a110be4b7617fbd9 2024-11-11T20:41:32,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/99de05a829174f9781844c6b0928606e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/99de05a829174f9781844c6b0928606e 2024-11-11T20:41:32,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d2f26fc62c29493fbcbd979ea9ea000a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/d2f26fc62c29493fbcbd979ea9ea000a 2024-11-11T20:41:32,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/92335b02d5ed4972b739b0dc2003fbbc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/92335b02d5ed4972b739b0dc2003fbbc 2024-11-11T20:41:32,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/086573b42abd47c0aa6fb8e6a18ed23f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/086573b42abd47c0aa6fb8e6a18ed23f 2024-11-11T20:41:32,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/bae9b0b4eaff49afbb18bb75c063c464 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/bae9b0b4eaff49afbb18bb75c063c464 2024-11-11T20:41:32,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0ff7fa2839f24a03b4d2591c1e4fbc86 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/0ff7fa2839f24a03b4d2591c1e4fbc86 2024-11-11T20:41:32,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051c4c5ef74c41d7b5f88e467bd8f279 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/051c4c5ef74c41d7b5f88e467bd8f279 2024-11-11T20:41:32,557 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/recovered.edits/396.seqid, newMaxSeqId=396, maxSeqId=4 2024-11-11T20:41:32,557 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. 2024-11-11T20:41:32,558 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for fe28f5753443ca2f28d81a625707f258: Waiting for close lock at 1731357690824Running coprocessor pre-close hooks at 1731357690824Disabling compacts and flushes for region at 1731357690824Disabling writes for close at 1731357690824Obtaining lock to block concurrent updates at 1731357690824Preparing flush snapshotting stores in fe28f5753443ca2f28d81a625707f258 at 1731357690824Finished memstore snapshotting TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731357690824Flushing stores of TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258. at 1731357690824Flushing fe28f5753443ca2f28d81a625707f258/A: creating writer at 1731357690824Flushing mob file fe28f5753443ca2f28d81a625707f258/A: appending metadata at 1731357690829 (+5 ms)Flushing mob file fe28f5753443ca2f28d81a625707f258/A: closing flushed file at 1731357690829Flushing fe28f5753443ca2f28d81a625707f258/A: appending metadata at 1731357691237 (+408 ms)Flushing fe28f5753443ca2f28d81a625707f258/A: closing flushed file at 1731357691237Flushing fe28f5753443ca2f28d81a625707f258/B: creating writer at 1731357691644 (+407 ms)Flushing fe28f5753443ca2f28d81a625707f258/B: appending metadata at 1731357691646 (+2 ms)Flushing fe28f5753443ca2f28d81a625707f258/B: closing flushed file at 1731357691646Flushing fe28f5753443ca2f28d81a625707f258/C: creating writer at 1731357692057 (+411 ms)Flushing fe28f5753443ca2f28d81a625707f258/C: appending metadata at 1731357692062 (+5 ms)Flushing fe28f5753443ca2f28d81a625707f258/C: closing flushed file at 1731357692062Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41a25f7d: reopening flushed file at 1731357692472 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6580463a: reopening flushed file at 1731357692477 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f176ab9: reopening flushed file at 1731357692481 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fe28f5753443ca2f28d81a625707f258 in 1662ms, sequenceid=393, compaction requested=true at 1731357692486 (+5 ms)Writing region close event to WAL at 1731357692555 (+69 ms)Running coprocessor post-close hooks at 1731357692557 (+2 ms)Closed at 1731357692557 2024-11-11T20:41:32,559 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,559 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=fe28f5753443ca2f28d81a625707f258, regionState=CLOSED 2024-11-11T20:41:32,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:41:32,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T20:41:32,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure fe28f5753443ca2f28d81a625707f258, server=51ca66f7ee3c,44271,1731357576703 in 1.8900 sec 2024-11-11T20:41:32,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-11T20:41:32,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe28f5753443ca2f28d81a625707f258, UNASSIGN in 1.8930 sec 2024-11-11T20:41:32,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T20:41:32,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8960 sec 2024-11-11T20:41:32,566 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357692566"}]},"ts":"1731357692566"} 2024-11-11T20:41:32,567 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:41:32,567 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:41:32,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9050 sec 2024-11-11T20:41:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:41:32,811 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:32,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:41:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:32,817 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T20:41:32,818 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:32,822 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,826 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/recovered.edits] 2024-11-11T20:41:32,830 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/42a7ddc9fe464923a56d515ef683a997 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/42a7ddc9fe464923a56d515ef683a997 2024-11-11T20:41:32,832 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4dd5c07b84b2431e91569a23f22da41c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/4dd5c07b84b2431e91569a23f22da41c 2024-11-11T20:41:32,835 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/e8a771f3ffe140e4b614d6bc9e729ad9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/A/e8a771f3ffe140e4b614d6bc9e729ad9 2024-11-11T20:41:32,839 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/0cb72f132967438985f46de6e981610e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/0cb72f132967438985f46de6e981610e 2024-11-11T20:41:32,841 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/1851673874fc49acb0b7033aeefe8a24 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/1851673874fc49acb0b7033aeefe8a24 2024-11-11T20:41:32,842 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/206bee059109481aa245b9cb0943df28 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/B/206bee059109481aa245b9cb0943df28 2024-11-11T20:41:32,846 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/6445b01c2eb84e00a6487fdbc3e4351f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/6445b01c2eb84e00a6487fdbc3e4351f 2024-11-11T20:41:32,848 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/93dec11c7fe242eb8e563fc26e3377dd to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/93dec11c7fe242eb8e563fc26e3377dd 2024-11-11T20:41:32,849 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/f211c2343e0c400089eeae66abce7d66 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/C/f211c2343e0c400089eeae66abce7d66 2024-11-11T20:41:32,853 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/recovered.edits/396.seqid to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258/recovered.edits/396.seqid 2024-11-11T20:41:32,853 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,853 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:41:32,854 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:41:32,855 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T20:41:32,860 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111197cefb96694bd0a884b1c2f1247255_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111197cefb96694bd0a884b1c2f1247255_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,862 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111a16150b62124e50bb69cec9f61a2ede_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111a16150b62124e50bb69cec9f61a2ede_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,864 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111129555c110dfc43eb91be03f52ba1cfcf_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111129555c110dfc43eb91be03f52ba1cfcf_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,865 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111138584baddcec48d7ac041bfff9668a86_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111138584baddcec48d7ac041bfff9668a86_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,867 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113e54a3772dcb429692b661d542519854_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113e54a3772dcb429692b661d542519854_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,869 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115c576df9325041f2b849a3786805c2e1_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115c576df9325041f2b849a3786805c2e1_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,870 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111174ab7a1370464e18a7f296e8774c0a9f_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111174ab7a1370464e18a7f296e8774c0a9f_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,872 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117817fa0b008b478b83f4640c6d64de6f_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117817fa0b008b478b83f4640c6d64de6f_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,873 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111192ab81587a4a4f84aa528675f5eb2916_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111192ab81587a4a4f84aa528675f5eb2916_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,875 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119dbcd4f53955448a97935a50399d8863_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119dbcd4f53955448a97935a50399d8863_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,877 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a3c9e71eaee345dcb230067c02e3d524_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a3c9e71eaee345dcb230067c02e3d524_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,879 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a3eb28a847ad40b2a2de2d09b1e0f195_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a3eb28a847ad40b2a2de2d09b1e0f195_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,881 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aeb664e7c03642ca9e59f32c0c1bb22e_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aeb664e7c03642ca9e59f32c0c1bb22e_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,882 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b192fe6cea424d04a3e76c5f8f12f5bc_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b192fe6cea424d04a3e76c5f8f12f5bc_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b36e1c41a7764f59b40ab74bd8806feb_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b36e1c41a7764f59b40ab74bd8806feb_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,885 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111be135255a4974d07bf2603ef2c8980d9_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111be135255a4974d07bf2603ef2c8980d9_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,886 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c099dfdb950d464a9acea3cd0ced3d83_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c099dfdb950d464a9acea3cd0ced3d83_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,887 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c766dad1d1a043f7b76173a82d472613_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c766dad1d1a043f7b76173a82d472613_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,888 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ca05fba6e7d64fc08feb200ef150b242_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ca05fba6e7d64fc08feb200ef150b242_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,888 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111faa457d487424064b55a9fdbf6c787ab_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111faa457d487424064b55a9fdbf6c787ab_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,889 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ffec2ff48890421d919cc5dcc05c07fb_fe28f5753443ca2f28d81a625707f258 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ffec2ff48890421d919cc5dcc05c07fb_fe28f5753443ca2f28d81a625707f258 2024-11-11T20:41:32,890 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:41:32,891 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:32,893 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:41:32,895 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:41:32,896 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:32,897 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:41:32,897 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731357692897"}]},"ts":"9223372036854775807"} 2024-11-11T20:41:32,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:41:32,899 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fe28f5753443ca2f28d81a625707f258, NAME => 'TestAcidGuarantees,,1731357665285.fe28f5753443ca2f28d81a625707f258.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:41:32,899 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:41:32,899 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731357692899"}]},"ts":"9223372036854775807"} 2024-11-11T20:41:32,900 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:41:32,902 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:32,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-11-11T20:41:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T20:41:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:41:32,921 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:32,928 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 240) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2095157064_22 at /127.0.0.1:33516 [Waiting for operation #382] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1228990331_22 at /127.0.0.1:52348 [Waiting for operation #1094] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1228990331_22 at /127.0.0.1:33508 [Waiting for operation #379] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2095157064_22 at /127.0.0.1:33514 [Waiting for operation #388] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 444) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4664 (was 4516) - AvailableMemoryMB LEAK? - 2024-11-11T20:41:32,935 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=4663 2024-11-11T20:41:32,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:41:32,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:41:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:41:32,937 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:41:32,938 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:32,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-11-11T20:41:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:41:32,939 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:41:32,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742329_1505 (size=959) 2024-11-11T20:41:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:41:33,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:41:33,345 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30 2024-11-11T20:41:33,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742330_1506 (size=53) 2024-11-11T20:41:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:41:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:41:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1be5d7c75f632327a07f28f72dd94fc1, disabling compactions & flushes 2024-11-11T20:41:33,750 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. after waiting 0 ms 2024-11-11T20:41:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:33,750 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1be5d7c75f632327a07f28f72dd94fc1: Waiting for close lock at 1731357693750Disabling compacts and flushes for region at 1731357693750Disabling writes for close at 1731357693750Writing region close event to WAL at 1731357693750Closed at 1731357693750 2024-11-11T20:41:33,751 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:41:33,751 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731357693751"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731357693751"}]},"ts":"1731357693751"} 2024-11-11T20:41:33,752 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:41:33,753 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:41:33,753 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357693753"}]},"ts":"1731357693753"} 2024-11-11T20:41:33,754 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:41:33,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1be5d7c75f632327a07f28f72dd94fc1, ASSIGN}] 2024-11-11T20:41:33,755 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1be5d7c75f632327a07f28f72dd94fc1, ASSIGN 2024-11-11T20:41:33,756 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1be5d7c75f632327a07f28f72dd94fc1, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44271,1731357576703; forceNewPlan=false, retain=false 2024-11-11T20:41:33,906 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=1be5d7c75f632327a07f28f72dd94fc1, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:33,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1be5d7c75f632327a07f28f72dd94fc1, ASSIGN because future has completed 2024-11-11T20:41:33,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:41:34,065 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:34,065 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:41:34,066 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,066 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:41:34,066 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,066 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,068 INFO [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,069 INFO [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1be5d7c75f632327a07f28f72dd94fc1 columnFamilyName A 2024-11-11T20:41:34,069 DEBUG [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:34,069 INFO [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] regionserver.HStore(327): Store=1be5d7c75f632327a07f28f72dd94fc1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:41:34,069 INFO [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,070 INFO [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1be5d7c75f632327a07f28f72dd94fc1 columnFamilyName B 2024-11-11T20:41:34,070 DEBUG [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:34,071 INFO [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] regionserver.HStore(327): Store=1be5d7c75f632327a07f28f72dd94fc1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:41:34,071 INFO [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:41:34,071 INFO [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1be5d7c75f632327a07f28f72dd94fc1 columnFamilyName C 2024-11-11T20:41:34,071 DEBUG [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:41:34,072 INFO [StoreOpener-1be5d7c75f632327a07f28f72dd94fc1-1 {}] regionserver.HStore(327): Store=1be5d7c75f632327a07f28f72dd94fc1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:41:34,072 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,073 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,073 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,074 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,074 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,074 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:41:34,075 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,076 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:41:34,077 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened 1be5d7c75f632327a07f28f72dd94fc1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73319678, jitterRate=0.09254834055900574}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:41:34,077 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:34,077 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for 1be5d7c75f632327a07f28f72dd94fc1: Running coprocessor pre-open hook at 1731357694066Writing region info on filesystem at 1731357694066Initializing all the Stores at 1731357694067 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357694067Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357694067Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357694067Cleaning up temporary data from old regions at 1731357694074 (+7 ms)Running coprocessor post-open hooks at 1731357694077 (+3 ms)Region opened successfully at 1731357694077 2024-11-11T20:41:34,078 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., pid=130, masterSystemTime=1731357694063 2024-11-11T20:41:34,079 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:34,079 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:34,080 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=1be5d7c75f632327a07f28f72dd94fc1, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:34,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:41:34,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T20:41:34,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure 1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 in 172 msec 2024-11-11T20:41:34,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-11T20:41:34,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1be5d7c75f632327a07f28f72dd94fc1, ASSIGN in 329 msec 2024-11-11T20:41:34,085 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:41:34,085 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357694085"}]},"ts":"1731357694085"} 2024-11-11T20:41:34,086 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:41:34,087 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:41:34,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-11T20:41:35,052 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:41:35,082 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:35,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,084 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,086 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17209dc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,090 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ac7eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,092 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,096 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:41:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,35885,1731357576039 2024-11-11T20:41:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6385c816 2024-11-11T20:41:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:41:35,099 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:41:35,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,101 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1396fd6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,103 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a5b693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,105 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@266b2839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,106 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69928a5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,108 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f1a6416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,109 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3feb2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,110 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dd624b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,112 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76384f41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,113 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2628698f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29e17fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,116 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b3dba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,117 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f3bd0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,118 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,118 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6a154d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,120 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa9894c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,121 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,121 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,122 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfd8b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,123 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260b9d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,125 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291e25d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,126 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625c0c80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:41:35,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:41:35,128 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:41:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:41:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:41:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d14d2c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:41:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:41:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:35,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7b3476, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:41:35,130 DEBUG [Thread-2255 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,130 DEBUG [Thread-2240 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,130 DEBUG [Thread-2251 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,130 DEBUG [Thread-2242 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,131 DEBUG [Thread-2244 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,131 DEBUG [Thread-2257 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,132 DEBUG [Thread-2253 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,132 DEBUG [Thread-2246 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,133 DEBUG [Thread-2259 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,134 DEBUG [Thread-2248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:41:35,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,137 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:35,137 DEBUG [Thread-2246 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-11T20:41:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T20:41:35,140 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:41:35,143 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:35,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:35,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2] 2024-11-11T20:41:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:35,150 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:35,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:41:35,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/693ddea08cff47e0b7c7bfd07bd7c005 is 50, key is test_row_0/A:col10/1731357695145/Put/seqid=0 2024-11-11T20:41:35,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742331_1507 (size=12001) 2024-11-11T20:41:35,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/693ddea08cff47e0b7c7bfd07bd7c005 2024-11-11T20:41:35,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357755191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357755197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/59cf414187f04822ac337dba21dae51e is 50, key is test_row_0/B:col10/1731357695145/Put/seqid=0 2024-11-11T20:41:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357755198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357755199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357755204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742332_1508 (size=12001) 2024-11-11T20:41:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T20:41:35,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T20:41:35,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:35,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357755311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357755311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357755321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357755321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357755321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T20:41:35,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:35,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T20:41:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357755521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357755521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357755530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357755530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357755530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T20:41:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/59cf414187f04822ac337dba21dae51e 2024-11-11T20:41:35,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/1862030a86fd4873824ba76242e7bed1 is 50, key is test_row_0/C:col10/1731357695145/Put/seqid=0 2024-11-11T20:41:35,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742333_1509 (size=12001) 2024-11-11T20:41:35,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T20:41:35,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T20:41:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357755831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357755831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357755841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357755841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357755841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:35,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T20:41:35,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:35,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:35,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:36,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/1862030a86fd4873824ba76242e7bed1 2024-11-11T20:41:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/693ddea08cff47e0b7c7bfd07bd7c005 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/693ddea08cff47e0b7c7bfd07bd7c005 2024-11-11T20:41:36,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/693ddea08cff47e0b7c7bfd07bd7c005, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:41:36,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/59cf414187f04822ac337dba21dae51e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/59cf414187f04822ac337dba21dae51e 2024-11-11T20:41:36,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/59cf414187f04822ac337dba21dae51e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:41:36,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T20:41:36,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/1862030a86fd4873824ba76242e7bed1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1862030a86fd4873824ba76242e7bed1 2024-11-11T20:41:36,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:36,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:36,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:36,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:36,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:36,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1862030a86fd4873824ba76242e7bed1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:41:36,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1be5d7c75f632327a07f28f72dd94fc1 in 916ms, sequenceid=13, compaction requested=false 2024-11-11T20:41:36,066 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T20:41:36,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:36,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T20:41:36,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:36,212 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:41:36,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/5ac7f0764582492dbfc8762ac27349b4 is 50, key is test_row_0/A:col10/1731357695198/Put/seqid=0 2024-11-11T20:41:36,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742334_1510 (size=12001) 2024-11-11T20:41:36,220 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/5ac7f0764582492dbfc8762ac27349b4 2024-11-11T20:41:36,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/ae45dce11c5e41229c22553dc34f67aa is 50, key is test_row_0/B:col10/1731357695198/Put/seqid=0 2024-11-11T20:41:36,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742335_1511 (size=12001) 2024-11-11T20:41:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T20:41:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:36,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:36,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357756355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357756360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357756361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357756362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357756362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357756471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357756480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357756481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357756481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357756481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,640 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/ae45dce11c5e41229c22553dc34f67aa 2024-11-11T20:41:36,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/3c8a502fa5954d1ba9f861da7971b7ae is 50, key is test_row_0/C:col10/1731357695198/Put/seqid=0 2024-11-11T20:41:36,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742336_1512 (size=12001) 2024-11-11T20:41:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357756680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357756690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357756700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357756701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357756701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357757001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357757010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357757010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357757020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357757020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,060 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/3c8a502fa5954d1ba9f861da7971b7ae 2024-11-11T20:41:37,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/5ac7f0764582492dbfc8762ac27349b4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/5ac7f0764582492dbfc8762ac27349b4 2024-11-11T20:41:37,069 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/5ac7f0764582492dbfc8762ac27349b4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T20:41:37,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/ae45dce11c5e41229c22553dc34f67aa as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/ae45dce11c5e41229c22553dc34f67aa 2024-11-11T20:41:37,074 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/ae45dce11c5e41229c22553dc34f67aa, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T20:41:37,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/3c8a502fa5954d1ba9f861da7971b7ae as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/3c8a502fa5954d1ba9f861da7971b7ae 2024-11-11T20:41:37,079 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/3c8a502fa5954d1ba9f861da7971b7ae, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T20:41:37,080 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1be5d7c75f632327a07f28f72dd94fc1 in 868ms, sequenceid=37, compaction requested=false 2024-11-11T20:41:37,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:37,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:37,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-11T20:41:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-11T20:41:37,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T20:41:37,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9380 sec 2024-11-11T20:41:37,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.9470 sec 2024-11-11T20:41:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T20:41:37,291 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:37,291 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-11T20:41:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T20:41:37,293 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:37,294 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:37,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T20:41:37,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T20:41:37,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:37,446 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:41:37,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/ccc5792d7e544c7194d2d73b2d659800 is 50, key is test_row_0/A:col10/1731357696361/Put/seqid=0 2024-11-11T20:41:37,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742337_1513 (size=12001) 2024-11-11T20:41:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:37,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:37,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357757554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357757555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357757562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357757563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357757564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T20:41:37,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357757670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357757671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357757671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357757681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357757681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,859 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/ccc5792d7e544c7194d2d73b2d659800 2024-11-11T20:41:37,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/0072efe73b0345f8a42d85c2f2282d62 is 50, key is test_row_0/B:col10/1731357696361/Put/seqid=0 2024-11-11T20:41:37,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742338_1514 (size=12001) 2024-11-11T20:41:37,872 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/0072efe73b0345f8a42d85c2f2282d62 2024-11-11T20:41:37,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/800e40b491c5418b86e4e2e35734490a is 50, key is test_row_0/C:col10/1731357696361/Put/seqid=0 2024-11-11T20:41:37,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742339_1515 (size=12001) 2024-11-11T20:41:37,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357757881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357757881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357757881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,893 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/800e40b491c5418b86e4e2e35734490a 2024-11-11T20:41:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/ccc5792d7e544c7194d2d73b2d659800 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ccc5792d7e544c7194d2d73b2d659800 2024-11-11T20:41:37,902 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ccc5792d7e544c7194d2d73b2d659800, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:41:37,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/0072efe73b0345f8a42d85c2f2282d62 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0072efe73b0345f8a42d85c2f2282d62 2024-11-11T20:41:37,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357757893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357757893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:37,907 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0072efe73b0345f8a42d85c2f2282d62, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:41:37,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/800e40b491c5418b86e4e2e35734490a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/800e40b491c5418b86e4e2e35734490a 2024-11-11T20:41:37,911 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/800e40b491c5418b86e4e2e35734490a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:41:37,911 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1be5d7c75f632327a07f28f72dd94fc1 in 465ms, sequenceid=49, compaction requested=true 2024-11-11T20:41:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-11T20:41:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-11T20:41:37,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T20:41:37,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 619 msec 2024-11-11T20:41:37,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 626 msec 2024-11-11T20:41:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T20:41:37,920 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:37,921 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-11T20:41:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T20:41:37,922 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:37,923 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:37,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T20:41:38,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T20:41:38,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:38,078 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T20:41:38,079 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:41:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/ffeeef842e034ca595a56ddec5d5bf56 is 50, key is test_row_0/A:col10/1731357697562/Put/seqid=0 2024-11-11T20:41:38,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742340_1516 (size=12001) 2024-11-11T20:41:38,094 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/ffeeef842e034ca595a56ddec5d5bf56 2024-11-11T20:41:38,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/aca30b54dc3c49ef992f4640465dfbb1 is 50, key is test_row_0/B:col10/1731357697562/Put/seqid=0 2024-11-11T20:41:38,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742341_1517 (size=12001) 2024-11-11T20:41:38,112 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/aca30b54dc3c49ef992f4640465dfbb1 2024-11-11T20:41:38,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/19039b1e84ef455885f51b37b1a3170d is 50, key is test_row_0/C:col10/1731357697562/Put/seqid=0 2024-11-11T20:41:38,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742342_1518 (size=12001) 2024-11-11T20:41:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:38,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T20:41:38,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357758249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357758249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357758249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357758250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357758250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357758371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357758371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357758371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357758371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357758371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,522 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/19039b1e84ef455885f51b37b1a3170d 2024-11-11T20:41:38,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/ffeeef842e034ca595a56ddec5d5bf56 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ffeeef842e034ca595a56ddec5d5bf56 2024-11-11T20:41:38,529 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ffeeef842e034ca595a56ddec5d5bf56, entries=150, sequenceid=74, filesize=11.7 K 2024-11-11T20:41:38,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/aca30b54dc3c49ef992f4640465dfbb1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/aca30b54dc3c49ef992f4640465dfbb1 2024-11-11T20:41:38,532 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/aca30b54dc3c49ef992f4640465dfbb1, entries=150, sequenceid=74, filesize=11.7 K 2024-11-11T20:41:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/19039b1e84ef455885f51b37b1a3170d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/19039b1e84ef455885f51b37b1a3170d 2024-11-11T20:41:38,537 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/19039b1e84ef455885f51b37b1a3170d, entries=150, sequenceid=74, filesize=11.7 K 2024-11-11T20:41:38,538 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1be5d7c75f632327a07f28f72dd94fc1 in 460ms, sequenceid=74, compaction requested=true 2024-11-11T20:41:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-11T20:41:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-11T20:41:38,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T20:41:38,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 616 msec 2024-11-11T20:41:38,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 620 msec 2024-11-11T20:41:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T20:41:38,551 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:38,551 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-11T20:41:38,553 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T20:41:38,553 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:38,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:38,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T20:41:38,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/b4bc567ab25d42f2869ae1a841c13502 is 50, key is test_row_1/A:col10/1731357698248/Put/seqid=0 2024-11-11T20:41:38,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742343_1519 (size=14337) 2024-11-11T20:41:38,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357758619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357758620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357758621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357758622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357758628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T20:41:38,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T20:41:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:38,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:38,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357758730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357758740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357758740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357758741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357758741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T20:41:38,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:38,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:38,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:38,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:38,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T20:41:38,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357758940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357758950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357758951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357758951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357758951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:38,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/b4bc567ab25d42f2869ae1a841c13502 2024-11-11T20:41:39,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/50a2fd6db4914140a607004cbfc19200 is 50, key is test_row_1/B:col10/1731357698248/Put/seqid=0 2024-11-11T20:41:39,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742344_1520 (size=9657) 2024-11-11T20:41:39,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T20:41:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:39,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T20:41:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T20:41:39,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357759250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357759260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357759261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357759261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357759271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T20:41:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:39,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:39,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/50a2fd6db4914140a607004cbfc19200 2024-11-11T20:41:39,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/a6f245a7b61d4e0b813584f6d73ecddb is 50, key is test_row_1/C:col10/1731357698248/Put/seqid=0 2024-11-11T20:41:39,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742345_1521 (size=9657) 2024-11-11T20:41:39,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/a6f245a7b61d4e0b813584f6d73ecddb 2024-11-11T20:41:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/b4bc567ab25d42f2869ae1a841c13502 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/b4bc567ab25d42f2869ae1a841c13502 2024-11-11T20:41:39,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/b4bc567ab25d42f2869ae1a841c13502, entries=200, sequenceid=88, filesize=14.0 K 2024-11-11T20:41:39,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/50a2fd6db4914140a607004cbfc19200 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/50a2fd6db4914140a607004cbfc19200 2024-11-11T20:41:39,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/50a2fd6db4914140a607004cbfc19200, entries=100, sequenceid=88, filesize=9.4 K 2024-11-11T20:41:39,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/a6f245a7b61d4e0b813584f6d73ecddb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/a6f245a7b61d4e0b813584f6d73ecddb 2024-11-11T20:41:39,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/a6f245a7b61d4e0b813584f6d73ecddb, entries=100, sequenceid=88, filesize=9.4 K 2024-11-11T20:41:39,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1be5d7c75f632327a07f28f72dd94fc1 in 851ms, sequenceid=88, compaction requested=true 2024-11-11T20:41:39,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:39,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:39,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:39,436 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T20:41:39,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:39,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:39,436 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T20:41:39,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:39,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:39,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T20:41:39,437 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62341 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T20:41:39,437 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/A is initiating minor compaction (all files) 2024-11-11T20:41:39,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/B is initiating minor compaction (all files) 2024-11-11T20:41:39,437 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/B in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,437 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/A in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,438 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/59cf414187f04822ac337dba21dae51e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/ae45dce11c5e41229c22553dc34f67aa, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0072efe73b0345f8a42d85c2f2282d62, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/aca30b54dc3c49ef992f4640465dfbb1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/50a2fd6db4914140a607004cbfc19200] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=56.3 K 2024-11-11T20:41:39,438 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/693ddea08cff47e0b7c7bfd07bd7c005, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/5ac7f0764582492dbfc8762ac27349b4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ccc5792d7e544c7194d2d73b2d659800, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ffeeef842e034ca595a56ddec5d5bf56, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/b4bc567ab25d42f2869ae1a841c13502] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=60.9 K 2024-11-11T20:41:39,440 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 59cf414187f04822ac337dba21dae51e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731357695145 2024-11-11T20:41:39,440 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693ddea08cff47e0b7c7bfd07bd7c005, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731357695145 2024-11-11T20:41:39,440 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ae45dce11c5e41229c22553dc34f67aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731357695190 2024-11-11T20:41:39,440 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac7f0764582492dbfc8762ac27349b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731357695190 2024-11-11T20:41:39,441 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0072efe73b0345f8a42d85c2f2282d62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357696359 2024-11-11T20:41:39,441 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc5792d7e544c7194d2d73b2d659800, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357696359 2024-11-11T20:41:39,441 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting aca30b54dc3c49ef992f4640465dfbb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731357697554 2024-11-11T20:41:39,441 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffeeef842e034ca595a56ddec5d5bf56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731357697554 2024-11-11T20:41:39,441 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a2fd6db4914140a607004cbfc19200, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731357698248 2024-11-11T20:41:39,442 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4bc567ab25d42f2869ae1a841c13502, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731357698248 2024-11-11T20:41:39,451 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#B#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:39,451 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#A#compaction#435 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:39,451 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/fe504ffb14e64e64a76c0147b5251abc is 50, key is test_row_0/B:col10/1731357697562/Put/seqid=0 2024-11-11T20:41:39,452 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/45037bc15d454ad8b57b4c92d509a30e is 50, key is test_row_0/A:col10/1731357697562/Put/seqid=0 2024-11-11T20:41:39,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742346_1522 (size=12173) 2024-11-11T20:41:39,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742347_1523 (size=12173) 2024-11-11T20:41:39,464 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/fe504ffb14e64e64a76c0147b5251abc as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe504ffb14e64e64a76c0147b5251abc 2024-11-11T20:41:39,464 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/45037bc15d454ad8b57b4c92d509a30e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45037bc15d454ad8b57b4c92d509a30e 2024-11-11T20:41:39,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T20:41:39,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,468 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:41:39,469 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/A of 1be5d7c75f632327a07f28f72dd94fc1 into 45037bc15d454ad8b57b4c92d509a30e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:39,469 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:39,469 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/A, priority=11, startTime=1731357699436; duration=0sec 2024-11-11T20:41:39,469 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/B of 1be5d7c75f632327a07f28f72dd94fc1 into fe504ffb14e64e64a76c0147b5251abc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:39,470 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:39,470 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:A 2024-11-11T20:41:39,470 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:39,470 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T20:41:39,470 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/B, priority=11, startTime=1731357699436; duration=0sec 2024-11-11T20:41:39,470 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:39,470 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:B 2024-11-11T20:41:39,471 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T20:41:39,471 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/C is initiating minor compaction (all files) 2024-11-11T20:41:39,471 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/C in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,472 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1862030a86fd4873824ba76242e7bed1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/3c8a502fa5954d1ba9f861da7971b7ae, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/800e40b491c5418b86e4e2e35734490a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/19039b1e84ef455885f51b37b1a3170d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/a6f245a7b61d4e0b813584f6d73ecddb] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=56.3 K 2024-11-11T20:41:39,472 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1862030a86fd4873824ba76242e7bed1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731357695145 2024-11-11T20:41:39,472 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c8a502fa5954d1ba9f861da7971b7ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731357695190 2024-11-11T20:41:39,472 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 800e40b491c5418b86e4e2e35734490a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731357696359 2024-11-11T20:41:39,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/bf3795304205436e92c6b5f649a9fa50 is 50, key is test_row_0/A:col10/1731357698626/Put/seqid=0 2024-11-11T20:41:39,473 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19039b1e84ef455885f51b37b1a3170d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731357697554 2024-11-11T20:41:39,473 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f245a7b61d4e0b813584f6d73ecddb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731357698248 2024-11-11T20:41:39,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742348_1524 (size=12001) 2024-11-11T20:41:39,491 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#C#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:41:39,491 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/7e076e71c9874fa6ac427d425d32435a is 50, key is test_row_0/C:col10/1731357697562/Put/seqid=0 2024-11-11T20:41:39,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742349_1525 (size=12173) 2024-11-11T20:41:39,580 WARN [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-11T20:41:39,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/693ddea08cff47e0b7c7bfd07bd7c005, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/5ac7f0764582492dbfc8762ac27349b4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ccc5792d7e544c7194d2d73b2d659800, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ffeeef842e034ca595a56ddec5d5bf56, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/b4bc567ab25d42f2869ae1a841c13502] to archive 2024-11-11T20:41:39,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:41:39,581 DEBUG [regionserver/51ca66f7ee3c:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2024-11-11T20:41:39,582 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/693ddea08cff47e0b7c7bfd07bd7c005 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/693ddea08cff47e0b7c7bfd07bd7c005 2024-11-11T20:41:39,583 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/5ac7f0764582492dbfc8762ac27349b4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/5ac7f0764582492dbfc8762ac27349b4 2024-11-11T20:41:39,586 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ccc5792d7e544c7194d2d73b2d659800 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ccc5792d7e544c7194d2d73b2d659800 2024-11-11T20:41:39,588 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ffeeef842e034ca595a56ddec5d5bf56 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ffeeef842e034ca595a56ddec5d5bf56 2024-11-11T20:41:39,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/b4bc567ab25d42f2869ae1a841c13502 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/b4bc567ab25d42f2869ae1a841c13502 2024-11-11T20:41:39,592 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/59cf414187f04822ac337dba21dae51e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/ae45dce11c5e41229c22553dc34f67aa, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0072efe73b0345f8a42d85c2f2282d62, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/aca30b54dc3c49ef992f4640465dfbb1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/50a2fd6db4914140a607004cbfc19200] to archive 2024-11-11T20:41:39,593 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:41:39,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/59cf414187f04822ac337dba21dae51e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/59cf414187f04822ac337dba21dae51e 2024-11-11T20:41:39,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/ae45dce11c5e41229c22553dc34f67aa to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/ae45dce11c5e41229c22553dc34f67aa 2024-11-11T20:41:39,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0072efe73b0345f8a42d85c2f2282d62 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0072efe73b0345f8a42d85c2f2282d62 2024-11-11T20:41:39,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/aca30b54dc3c49ef992f4640465dfbb1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/aca30b54dc3c49ef992f4640465dfbb1 2024-11-11T20:41:39,598 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/50a2fd6db4914140a607004cbfc19200 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/50a2fd6db4914140a607004cbfc19200 2024-11-11T20:41:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T20:41:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:39,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:39,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357759792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357759800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357759801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357759801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357759802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,888 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/bf3795304205436e92c6b5f649a9fa50 2024-11-11T20:41:39,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/26332e34c3404842a0a196c1539efe3c is 50, key is test_row_0/B:col10/1731357698626/Put/seqid=0 2024-11-11T20:41:39,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742350_1526 (size=12001) 2024-11-11T20:41:39,898 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/26332e34c3404842a0a196c1539efe3c 2024-11-11T20:41:39,900 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/7e076e71c9874fa6ac427d425d32435a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7e076e71c9874fa6ac427d425d32435a 2024-11-11T20:41:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/e6fc609b5b5b41bda386425cc717a9f4 is 50, key is test_row_0/C:col10/1731357698626/Put/seqid=0 2024-11-11T20:41:39,907 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/C of 1be5d7c75f632327a07f28f72dd94fc1 into 7e076e71c9874fa6ac427d425d32435a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:39,907 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:39,907 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/C, priority=11, startTime=1731357699436; duration=0sec 2024-11-11T20:41:39,907 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:39,907 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:C 2024-11-11T20:41:39,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742351_1527 (size=12001) 2024-11-11T20:41:39,919 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/e6fc609b5b5b41bda386425cc717a9f4 2024-11-11T20:41:39,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357759911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357759911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/bf3795304205436e92c6b5f649a9fa50 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/bf3795304205436e92c6b5f649a9fa50 2024-11-11T20:41:39,926 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/bf3795304205436e92c6b5f649a9fa50, entries=150, sequenceid=110, filesize=11.7 K 2024-11-11T20:41:39,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/26332e34c3404842a0a196c1539efe3c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/26332e34c3404842a0a196c1539efe3c 2024-11-11T20:41:39,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357759921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357759921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357759921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:39,930 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/26332e34c3404842a0a196c1539efe3c, entries=150, sequenceid=110, filesize=11.7 K 2024-11-11T20:41:39,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/e6fc609b5b5b41bda386425cc717a9f4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/e6fc609b5b5b41bda386425cc717a9f4 2024-11-11T20:41:39,937 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/e6fc609b5b5b41bda386425cc717a9f4, entries=150, sequenceid=110, filesize=11.7 K 2024-11-11T20:41:39,938 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1be5d7c75f632327a07f28f72dd94fc1 in 470ms, sequenceid=110, compaction requested=false 2024-11-11T20:41:39,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:39,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:39,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-11T20:41:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-11T20:41:39,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T20:41:39,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3870 sec 2024-11-11T20:41:39,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.3930 sec 2024-11-11T20:41:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:40,135 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:40,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T20:41:40,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/c7ac20001020422ebaf3b69ea159eee7 is 50, key is test_row_0/A:col10/1731357700134/Put/seqid=0 2024-11-11T20:41:40,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742352_1528 (size=14341) 2024-11-11T20:41:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357760168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357760169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357760170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357760171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357760175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357760281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357760291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357760291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357760291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357760292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357760491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357760511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357760511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357760511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357760511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/c7ac20001020422ebaf3b69ea159eee7 2024-11-11T20:41:40,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/843fdae3c65f4aadaa15c5e91dc72f3a is 50, key is test_row_0/B:col10/1731357700134/Put/seqid=0 2024-11-11T20:41:40,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742353_1529 (size=12001) 2024-11-11T20:41:40,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/843fdae3c65f4aadaa15c5e91dc72f3a 2024-11-11T20:41:40,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/dc9a1d0784974bfeb95ed55dc912259a is 50, key is test_row_0/C:col10/1731357700134/Put/seqid=0 2024-11-11T20:41:40,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742354_1530 (size=12001) 2024-11-11T20:41:40,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/dc9a1d0784974bfeb95ed55dc912259a 2024-11-11T20:41:40,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/c7ac20001020422ebaf3b69ea159eee7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c7ac20001020422ebaf3b69ea159eee7 2024-11-11T20:41:40,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c7ac20001020422ebaf3b69ea159eee7, entries=200, sequenceid=128, filesize=14.0 K 2024-11-11T20:41:40,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/843fdae3c65f4aadaa15c5e91dc72f3a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/843fdae3c65f4aadaa15c5e91dc72f3a 2024-11-11T20:41:40,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/843fdae3c65f4aadaa15c5e91dc72f3a, entries=150, sequenceid=128, filesize=11.7 K 2024-11-11T20:41:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/dc9a1d0784974bfeb95ed55dc912259a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/dc9a1d0784974bfeb95ed55dc912259a 2024-11-11T20:41:40,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/dc9a1d0784974bfeb95ed55dc912259a, entries=150, sequenceid=128, filesize=11.7 K 2024-11-11T20:41:40,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1be5d7c75f632327a07f28f72dd94fc1 in 472ms, sequenceid=128, compaction requested=true 2024-11-11T20:41:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:40,608 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:40,608 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:40,608 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:40,608 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:40,608 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/B is initiating minor compaction (all files) 2024-11-11T20:41:40,608 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/A is initiating minor compaction (all files) 2024-11-11T20:41:40,609 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/B in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:40,609 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/A in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:40,609 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe504ffb14e64e64a76c0147b5251abc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/26332e34c3404842a0a196c1539efe3c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/843fdae3c65f4aadaa15c5e91dc72f3a] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=35.3 K 2024-11-11T20:41:40,609 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45037bc15d454ad8b57b4c92d509a30e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/bf3795304205436e92c6b5f649a9fa50, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c7ac20001020422ebaf3b69ea159eee7] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=37.6 K 2024-11-11T20:41:40,609 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fe504ffb14e64e64a76c0147b5251abc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731357697562 2024-11-11T20:41:40,609 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45037bc15d454ad8b57b4c92d509a30e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731357697562 2024-11-11T20:41:40,609 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 26332e34c3404842a0a196c1539efe3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731357698618 2024-11-11T20:41:40,609 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf3795304205436e92c6b5f649a9fa50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731357698618 2024-11-11T20:41:40,609 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 843fdae3c65f4aadaa15c5e91dc72f3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731357699799 2024-11-11T20:41:40,610 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7ac20001020422ebaf3b69ea159eee7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731357699799 2024-11-11T20:41:40,618 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:40,619 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#B#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:40,619 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/c9675ae8b6e84a4b8606275a09a22ccb is 50, key is test_row_0/A:col10/1731357700134/Put/seqid=0 2024-11-11T20:41:40,619 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/561f3c8185be4dafbee7946d3e819d1e is 50, key is test_row_0/B:col10/1731357700134/Put/seqid=0 2024-11-11T20:41:40,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742355_1531 (size=12104) 2024-11-11T20:41:40,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742356_1532 (size=12104) 2024-11-11T20:41:40,628 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/c9675ae8b6e84a4b8606275a09a22ccb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c9675ae8b6e84a4b8606275a09a22ccb 2024-11-11T20:41:40,634 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/561f3c8185be4dafbee7946d3e819d1e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/561f3c8185be4dafbee7946d3e819d1e 2024-11-11T20:41:40,636 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/A of 1be5d7c75f632327a07f28f72dd94fc1 into c9675ae8b6e84a4b8606275a09a22ccb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:40,636 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:40,636 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/A, priority=13, startTime=1731357700607; duration=0sec 2024-11-11T20:41:40,636 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:40,636 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:A 2024-11-11T20:41:40,636 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:40,646 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:40,646 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/C is initiating minor compaction (all files) 2024-11-11T20:41:40,646 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/C in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:40,646 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7e076e71c9874fa6ac427d425d32435a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/e6fc609b5b5b41bda386425cc717a9f4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/dc9a1d0784974bfeb95ed55dc912259a] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=35.3 K 2024-11-11T20:41:40,647 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e076e71c9874fa6ac427d425d32435a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731357697562 2024-11-11T20:41:40,647 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6fc609b5b5b41bda386425cc717a9f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731357698618 2024-11-11T20:41:40,648 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/B of 1be5d7c75f632327a07f28f72dd94fc1 into 561f3c8185be4dafbee7946d3e819d1e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:40,648 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:40,648 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/B, priority=13, startTime=1731357700608; duration=0sec 2024-11-11T20:41:40,648 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:40,648 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:B 2024-11-11T20:41:40,648 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc9a1d0784974bfeb95ed55dc912259a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731357699799 2024-11-11T20:41:40,662 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:40,663 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/c90a512b0d0b43a3a99622aedfc61391 is 50, key is test_row_0/C:col10/1731357700134/Put/seqid=0 2024-11-11T20:41:40,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742357_1533 (size=12275) 2024-11-11T20:41:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T20:41:40,701 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:40,702 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-11T20:41:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T20:41:40,704 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:40,705 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:40,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:40,803 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:40,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:41:40,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/ae0e5f65e45543d790eb4dc9a12acb78 is 50, key is test_row_0/A:col10/1731357700802/Put/seqid=0 2024-11-11T20:41:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T20:41:40,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742358_1534 (size=14541) 2024-11-11T20:41:40,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/ae0e5f65e45543d790eb4dc9a12acb78 2024-11-11T20:41:40,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/16a22010030e44d19a6be1febbb0359f is 50, key is test_row_0/B:col10/1731357700802/Put/seqid=0 2024-11-11T20:41:40,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742359_1535 (size=12151) 2024-11-11T20:41:40,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357760837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357760838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357760839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357760839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357760840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T20:41:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:40,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:40,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357760950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357760951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357760951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357760951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:40,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357760961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T20:41:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T20:41:41,080 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/c90a512b0d0b43a3a99622aedfc61391 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c90a512b0d0b43a3a99622aedfc61391 2024-11-11T20:41:41,083 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/C of 1be5d7c75f632327a07f28f72dd94fc1 into c90a512b0d0b43a3a99622aedfc61391(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:41,083 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:41,083 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/C, priority=13, startTime=1731357700608; duration=0sec 2024-11-11T20:41:41,083 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:41,084 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:C 2024-11-11T20:41:41,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T20:41:41,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:41,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357761161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357761161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357761161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357761170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357761170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/16a22010030e44d19a6be1febbb0359f 2024-11-11T20:41:41,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/920edd3d323b4eb3b4796cd449523a0b is 50, key is test_row_0/C:col10/1731357700802/Put/seqid=0 2024-11-11T20:41:41,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742360_1536 (size=12151) 2024-11-11T20:41:41,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T20:41:41,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T20:41:41,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T20:41:41,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:41,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357761470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357761481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357761481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357761481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357761481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:41,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T20:41:41,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:41,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:41,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/920edd3d323b4eb3b4796cd449523a0b 2024-11-11T20:41:41,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/ae0e5f65e45543d790eb4dc9a12acb78 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ae0e5f65e45543d790eb4dc9a12acb78 2024-11-11T20:41:41,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ae0e5f65e45543d790eb4dc9a12acb78, entries=200, sequenceid=152, filesize=14.2 K 2024-11-11T20:41:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/16a22010030e44d19a6be1febbb0359f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/16a22010030e44d19a6be1febbb0359f 2024-11-11T20:41:41,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/16a22010030e44d19a6be1febbb0359f, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T20:41:41,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/920edd3d323b4eb3b4796cd449523a0b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/920edd3d323b4eb3b4796cd449523a0b 2024-11-11T20:41:41,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/920edd3d323b4eb3b4796cd449523a0b, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T20:41:41,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1be5d7c75f632327a07f28f72dd94fc1 in 855ms, sequenceid=152, compaction requested=false 2024-11-11T20:41:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:41,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T20:41:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:41,771 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T20:41:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/6f63557d2ca247d1b55510df5fa14ea6 is 50, key is test_row_0/A:col10/1731357700839/Put/seqid=0 2024-11-11T20:41:41,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742361_1537 (size=12151) 2024-11-11T20:41:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T20:41:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:41,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:42,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357762026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357762026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357762027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357762030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357762033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357762141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357762141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357762141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357762141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357762150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,187 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/6f63557d2ca247d1b55510df5fa14ea6 2024-11-11T20:41:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7e13d728a7f0486bb61aa2cc3aa6c0d8 is 50, key is test_row_0/B:col10/1731357700839/Put/seqid=0 2024-11-11T20:41:42,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742362_1538 (size=12151) 2024-11-11T20:41:42,201 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7e13d728a7f0486bb61aa2cc3aa6c0d8 2024-11-11T20:41:42,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/bbbea0213cf440268ba57ddad3c90854 is 50, key is test_row_0/C:col10/1731357700839/Put/seqid=0 2024-11-11T20:41:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742363_1539 (size=12151) 2024-11-11T20:41:42,234 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/bbbea0213cf440268ba57ddad3c90854 2024-11-11T20:41:42,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/6f63557d2ca247d1b55510df5fa14ea6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/6f63557d2ca247d1b55510df5fa14ea6 2024-11-11T20:41:42,252 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/6f63557d2ca247d1b55510df5fa14ea6, entries=150, sequenceid=167, filesize=11.9 K 2024-11-11T20:41:42,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7e13d728a7f0486bb61aa2cc3aa6c0d8 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e13d728a7f0486bb61aa2cc3aa6c0d8 2024-11-11T20:41:42,262 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e13d728a7f0486bb61aa2cc3aa6c0d8, entries=150, sequenceid=167, filesize=11.9 K 2024-11-11T20:41:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/bbbea0213cf440268ba57ddad3c90854 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/bbbea0213cf440268ba57ddad3c90854 2024-11-11T20:41:42,269 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/bbbea0213cf440268ba57ddad3c90854, entries=150, sequenceid=167, filesize=11.9 K 2024-11-11T20:41:42,269 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1be5d7c75f632327a07f28f72dd94fc1 in 498ms, sequenceid=167, compaction requested=true 2024-11-11T20:41:42,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:42,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:42,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-11T20:41:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-11T20:41:42,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T20:41:42,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5650 sec 2024-11-11T20:41:42,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.5710 sec 2024-11-11T20:41:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:42,354 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:42,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T20:41:42,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/cbc95b4241574f768bd62a310d576bb7 is 50, key is test_row_0/A:col10/1731357702353/Put/seqid=0 2024-11-11T20:41:42,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742364_1540 (size=14541) 2024-11-11T20:41:42,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/cbc95b4241574f768bd62a310d576bb7 2024-11-11T20:41:42,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/f46eba518cbf4373a3a153899d78f420 is 50, key is test_row_0/B:col10/1731357702353/Put/seqid=0 2024-11-11T20:41:42,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357762368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357762369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357762370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357762371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357762376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742365_1541 (size=12151) 2024-11-11T20:41:42,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357762481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357762481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357762481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357762481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357762490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357762690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357762700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357762700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357762701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357762711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:42,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/f46eba518cbf4373a3a153899d78f420 2024-11-11T20:41:42,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/355bcdea86e040098fa8b65b04737476 is 50, key is test_row_0/C:col10/1731357702353/Put/seqid=0 2024-11-11T20:41:42,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742366_1542 (size=12151) 2024-11-11T20:41:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T20:41:42,851 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:42,852 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-11T20:41:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T20:41:42,853 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:42,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:42,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T20:41:43,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357763000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-11T20:41:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:43,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:43,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357763010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357763021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357763021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357763021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-11T20:41:43,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:43,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:43,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:43,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:43,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T20:41:43,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/355bcdea86e040098fa8b65b04737476 2024-11-11T20:41:43,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/cbc95b4241574f768bd62a310d576bb7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cbc95b4241574f768bd62a310d576bb7 2024-11-11T20:41:43,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cbc95b4241574f768bd62a310d576bb7, entries=200, sequenceid=190, filesize=14.2 K 2024-11-11T20:41:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/f46eba518cbf4373a3a153899d78f420 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f46eba518cbf4373a3a153899d78f420 2024-11-11T20:41:43,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f46eba518cbf4373a3a153899d78f420, entries=150, sequenceid=190, filesize=11.9 K 2024-11-11T20:41:43,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/355bcdea86e040098fa8b65b04737476 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/355bcdea86e040098fa8b65b04737476 2024-11-11T20:41:43,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/355bcdea86e040098fa8b65b04737476, entries=150, sequenceid=190, filesize=11.9 K 2024-11-11T20:41:43,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1be5d7c75f632327a07f28f72dd94fc1 in 860ms, sequenceid=190, compaction requested=true 2024-11-11T20:41:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:43,214 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:43,214 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:43,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:43,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:43,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:43,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:43,217 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53337 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:43,217 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48557 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:43,217 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/A is initiating minor compaction (all files) 2024-11-11T20:41:43,217 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/B is initiating minor compaction (all files) 2024-11-11T20:41:43,217 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/B in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:43,217 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/A in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:43,217 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/561f3c8185be4dafbee7946d3e819d1e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/16a22010030e44d19a6be1febbb0359f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e13d728a7f0486bb61aa2cc3aa6c0d8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f46eba518cbf4373a3a153899d78f420] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=47.4 K 2024-11-11T20:41:43,217 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c9675ae8b6e84a4b8606275a09a22ccb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ae0e5f65e45543d790eb4dc9a12acb78, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/6f63557d2ca247d1b55510df5fa14ea6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cbc95b4241574f768bd62a310d576bb7] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=52.1 K 2024-11-11T20:41:43,217 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 561f3c8185be4dafbee7946d3e819d1e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731357699799 2024-11-11T20:41:43,217 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9675ae8b6e84a4b8606275a09a22ccb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731357699799 2024-11-11T20:41:43,217 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a22010030e44d19a6be1febbb0359f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731357700170 2024-11-11T20:41:43,217 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0e5f65e45543d790eb4dc9a12acb78, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731357700169 2024-11-11T20:41:43,218 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e13d728a7f0486bb61aa2cc3aa6c0d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731357700837 2024-11-11T20:41:43,218 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f63557d2ca247d1b55510df5fa14ea6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731357700837 2024-11-11T20:41:43,218 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f46eba518cbf4373a3a153899d78f420, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731357702025 2024-11-11T20:41:43,218 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc95b4241574f768bd62a310d576bb7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731357702025 2024-11-11T20:41:43,224 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:43,224 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#A#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:43,225 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7c06fd0484b54285af1a45957e4c7c0e is 50, key is test_row_0/B:col10/1731357702353/Put/seqid=0 2024-11-11T20:41:43,225 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/a96a0f5983134854b6d747e6b51753b4 is 50, key is test_row_0/A:col10/1731357702353/Put/seqid=0 2024-11-11T20:41:43,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742367_1543 (size=12391) 2024-11-11T20:41:43,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742368_1544 (size=12391) 2024-11-11T20:41:43,232 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/a96a0f5983134854b6d747e6b51753b4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/a96a0f5983134854b6d747e6b51753b4 2024-11-11T20:41:43,236 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/A of 1be5d7c75f632327a07f28f72dd94fc1 into a96a0f5983134854b6d747e6b51753b4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:43,236 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:43,236 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/A, priority=12, startTime=1731357703214; duration=0sec 2024-11-11T20:41:43,236 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:43,236 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:A 2024-11-11T20:41:43,236 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:43,237 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:43,237 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/C is initiating minor compaction (all files) 2024-11-11T20:41:43,237 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/C in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:43,238 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c90a512b0d0b43a3a99622aedfc61391, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/920edd3d323b4eb3b4796cd449523a0b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/bbbea0213cf440268ba57ddad3c90854, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/355bcdea86e040098fa8b65b04737476] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=47.6 K 2024-11-11T20:41:43,238 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c90a512b0d0b43a3a99622aedfc61391, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731357699799 2024-11-11T20:41:43,238 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 920edd3d323b4eb3b4796cd449523a0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731357700170 2024-11-11T20:41:43,238 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbbea0213cf440268ba57ddad3c90854, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731357700837 2024-11-11T20:41:43,239 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 355bcdea86e040098fa8b65b04737476, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731357702025 2024-11-11T20:41:43,246 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#C#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:43,246 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/7fe3a8e3b28743b39d3983d9f184b516 is 50, key is test_row_0/C:col10/1731357702353/Put/seqid=0 2024-11-11T20:41:43,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742369_1545 (size=12561) 2024-11-11T20:41:43,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-11T20:41:43,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:43,312 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T20:41:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/7ef9c4e96dbb4a47a4b92fcead8ce360 is 50, key is test_row_0/A:col10/1731357702375/Put/seqid=0 2024-11-11T20:41:43,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742370_1546 (size=9757) 2024-11-11T20:41:43,318 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/7ef9c4e96dbb4a47a4b92fcead8ce360 2024-11-11T20:41:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/e3e557a07d0d481d95fa1066b23ed7eb is 50, key is test_row_0/B:col10/1731357702375/Put/seqid=0 2024-11-11T20:41:43,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742371_1547 (size=9757) 2024-11-11T20:41:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T20:41:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:43,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357763561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357763562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357763563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357763568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357763569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,632 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7c06fd0484b54285af1a45957e4c7c0e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7c06fd0484b54285af1a45957e4c7c0e 2024-11-11T20:41:43,636 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/B of 1be5d7c75f632327a07f28f72dd94fc1 into 7c06fd0484b54285af1a45957e4c7c0e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:43,637 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:43,637 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/B, priority=12, startTime=1731357703214; duration=0sec 2024-11-11T20:41:43,637 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:43,637 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:B 2024-11-11T20:41:43,663 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/7fe3a8e3b28743b39d3983d9f184b516 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7fe3a8e3b28743b39d3983d9f184b516 2024-11-11T20:41:43,670 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/C of 1be5d7c75f632327a07f28f72dd94fc1 into 7fe3a8e3b28743b39d3983d9f184b516(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:43,670 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:43,670 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/C, priority=12, startTime=1731357703215; duration=0sec 2024-11-11T20:41:43,670 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:43,670 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:C 2024-11-11T20:41:43,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357763670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357763681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357763681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357763681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357763681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,729 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/e3e557a07d0d481d95fa1066b23ed7eb 2024-11-11T20:41:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/d863d59cb1244a9b80a9d4d3bd853029 is 50, key is test_row_0/C:col10/1731357702375/Put/seqid=0 2024-11-11T20:41:43,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742372_1548 (size=9757) 2024-11-11T20:41:43,756 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/d863d59cb1244a9b80a9d4d3bd853029 2024-11-11T20:41:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/7ef9c4e96dbb4a47a4b92fcead8ce360 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ef9c4e96dbb4a47a4b92fcead8ce360 2024-11-11T20:41:43,766 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ef9c4e96dbb4a47a4b92fcead8ce360, entries=100, sequenceid=204, filesize=9.5 K 2024-11-11T20:41:43,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/e3e557a07d0d481d95fa1066b23ed7eb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/e3e557a07d0d481d95fa1066b23ed7eb 2024-11-11T20:41:43,770 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/e3e557a07d0d481d95fa1066b23ed7eb, entries=100, sequenceid=204, filesize=9.5 K 2024-11-11T20:41:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/d863d59cb1244a9b80a9d4d3bd853029 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d863d59cb1244a9b80a9d4d3bd853029 2024-11-11T20:41:43,775 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d863d59cb1244a9b80a9d4d3bd853029, entries=100, sequenceid=204, filesize=9.5 K 2024-11-11T20:41:43,776 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1be5d7c75f632327a07f28f72dd94fc1 in 465ms, sequenceid=204, compaction requested=false 2024-11-11T20:41:43,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:43,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:43,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-11T20:41:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-11T20:41:43,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T20:41:43,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 925 msec 2024-11-11T20:41:43,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 929 msec 2024-11-11T20:41:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:43,898 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:43,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T20:41:43,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/03264dfc529a44778b36aaf1b1262f2c is 50, key is test_row_0/A:col10/1731357703896/Put/seqid=0 2024-11-11T20:41:43,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742373_1549 (size=14541) 2024-11-11T20:41:43,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/03264dfc529a44778b36aaf1b1262f2c 2024-11-11T20:41:43,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/b505008e6632416f9b579f3ac72863a1 is 50, key is test_row_0/B:col10/1731357703896/Put/seqid=0 2024-11-11T20:41:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357763911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357763913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742374_1550 (size=12151) 2024-11-11T20:41:43,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357763916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/b505008e6632416f9b579f3ac72863a1 2024-11-11T20:41:43,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/9f0276987b944ae29ff6f03bfd0dfda2 is 50, key is test_row_0/C:col10/1731357703896/Put/seqid=0 2024-11-11T20:41:43,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357763925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357763926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:43,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742375_1551 (size=12151) 2024-11-11T20:41:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T20:41:43,991 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:43,991 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-11T20:41:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T20:41:43,993 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:43,994 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:43,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:44,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357764030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357764030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357764031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357764041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357764041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T20:41:44,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-11T20:41:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:44,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:44,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357764241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357764241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357764251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357764260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357764260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-11T20:41:44,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:44,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:44,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:44,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:44,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T20:41:44,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/9f0276987b944ae29ff6f03bfd0dfda2 2024-11-11T20:41:44,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/03264dfc529a44778b36aaf1b1262f2c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/03264dfc529a44778b36aaf1b1262f2c 2024-11-11T20:41:44,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/03264dfc529a44778b36aaf1b1262f2c, entries=200, sequenceid=232, filesize=14.2 K 2024-11-11T20:41:44,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/b505008e6632416f9b579f3ac72863a1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/b505008e6632416f9b579f3ac72863a1 2024-11-11T20:41:44,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/b505008e6632416f9b579f3ac72863a1, entries=150, sequenceid=232, filesize=11.9 K 2024-11-11T20:41:44,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/9f0276987b944ae29ff6f03bfd0dfda2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/9f0276987b944ae29ff6f03bfd0dfda2 2024-11-11T20:41:44,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/9f0276987b944ae29ff6f03bfd0dfda2, entries=150, sequenceid=232, filesize=11.9 K 2024-11-11T20:41:44,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1be5d7c75f632327a07f28f72dd94fc1 in 462ms, sequenceid=232, compaction requested=true 2024-11-11T20:41:44,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:44,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:44,360 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:44,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:44,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:44,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:44,360 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:44,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:44,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:44,361 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:44,361 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:44,361 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/B is initiating minor compaction (all files) 2024-11-11T20:41:44,361 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/A is initiating minor compaction (all files) 2024-11-11T20:41:44,361 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/B in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:44,361 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/A in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:44,361 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7c06fd0484b54285af1a45957e4c7c0e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/e3e557a07d0d481d95fa1066b23ed7eb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/b505008e6632416f9b579f3ac72863a1] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=33.5 K 2024-11-11T20:41:44,361 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/a96a0f5983134854b6d747e6b51753b4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ef9c4e96dbb4a47a4b92fcead8ce360, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/03264dfc529a44778b36aaf1b1262f2c] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=35.8 K 2024-11-11T20:41:44,361 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c06fd0484b54285af1a45957e4c7c0e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731357702025 2024-11-11T20:41:44,361 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a96a0f5983134854b6d747e6b51753b4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731357702025 2024-11-11T20:41:44,362 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ef9c4e96dbb4a47a4b92fcead8ce360, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731357702369 2024-11-11T20:41:44,362 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e557a07d0d481d95fa1066b23ed7eb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731357702369 2024-11-11T20:41:44,362 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03264dfc529a44778b36aaf1b1262f2c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731357703561 2024-11-11T20:41:44,362 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b505008e6632416f9b579f3ac72863a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731357703561 2024-11-11T20:41:44,368 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:44,368 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#A#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:44,368 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/0629bfc86317426a9946d57395b007ff is 50, key is test_row_0/B:col10/1731357703896/Put/seqid=0 2024-11-11T20:41:44,368 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/557b0c1f4d004f5aa07ff42a53bebf98 is 50, key is test_row_0/A:col10/1731357703896/Put/seqid=0 2024-11-11T20:41:44,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742376_1552 (size=12493) 2024-11-11T20:41:44,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742377_1553 (size=12493) 2024-11-11T20:41:44,381 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/0629bfc86317426a9946d57395b007ff as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0629bfc86317426a9946d57395b007ff 2024-11-11T20:41:44,385 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/B of 1be5d7c75f632327a07f28f72dd94fc1 into 0629bfc86317426a9946d57395b007ff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:44,385 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:44,385 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/B, priority=13, startTime=1731357704360; duration=0sec 2024-11-11T20:41:44,385 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:44,385 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:B 2024-11-11T20:41:44,385 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:44,387 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:44,387 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/C is initiating minor compaction (all files) 2024-11-11T20:41:44,387 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/C in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:44,387 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7fe3a8e3b28743b39d3983d9f184b516, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d863d59cb1244a9b80a9d4d3bd853029, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/9f0276987b944ae29ff6f03bfd0dfda2] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=33.7 K 2024-11-11T20:41:44,387 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe3a8e3b28743b39d3983d9f184b516, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731357702025 2024-11-11T20:41:44,387 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting d863d59cb1244a9b80a9d4d3bd853029, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731357702369 2024-11-11T20:41:44,388 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f0276987b944ae29ff6f03bfd0dfda2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731357703561 2024-11-11T20:41:44,393 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:44,394 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/258bbf78c89f4e6ab04b4777b4db21bb is 50, key is test_row_0/C:col10/1731357703896/Put/seqid=0 2024-11-11T20:41:44,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742378_1554 (size=12663) 2024-11-11T20:41:44,412 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/258bbf78c89f4e6ab04b4777b4db21bb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/258bbf78c89f4e6ab04b4777b4db21bb 2024-11-11T20:41:44,416 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/C of 1be5d7c75f632327a07f28f72dd94fc1 into 258bbf78c89f4e6ab04b4777b4db21bb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:44,417 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:44,417 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/C, priority=13, startTime=1731357704360; duration=0sec 2024-11-11T20:41:44,417 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:44,417 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:C 2024-11-11T20:41:44,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-11T20:41:44,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:44,451 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:41:44,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/cdaff4cb82554d05bbfb7f457abe0225 is 50, key is test_row_0/A:col10/1731357703921/Put/seqid=0 2024-11-11T20:41:44,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742379_1555 (size=12151) 2024-11-11T20:41:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:44,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357764595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357764596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357764596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357764597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357764598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T20:41:44,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357764700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357764710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357764710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357764710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357764711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,781 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/557b0c1f4d004f5aa07ff42a53bebf98 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/557b0c1f4d004f5aa07ff42a53bebf98 2024-11-11T20:41:44,785 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/A of 1be5d7c75f632327a07f28f72dd94fc1 into 557b0c1f4d004f5aa07ff42a53bebf98(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:44,785 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:44,785 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/A, priority=13, startTime=1731357704360; duration=0sec 2024-11-11T20:41:44,785 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:44,785 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:A 2024-11-11T20:41:44,866 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/cdaff4cb82554d05bbfb7f457abe0225 2024-11-11T20:41:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7ab13ecf91de4c17b2d93b1db6328fed is 50, key is test_row_0/B:col10/1731357703921/Put/seqid=0 2024-11-11T20:41:44,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742380_1556 (size=12151) 2024-11-11T20:41:44,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357764911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357764920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357764920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357764920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357764921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T20:41:45,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357765221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357765231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357765231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357765231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357765240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,274 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7ab13ecf91de4c17b2d93b1db6328fed 2024-11-11T20:41:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/aa610fe7d41b40dcb3b325379b9abf14 is 50, key is test_row_0/C:col10/1731357703921/Put/seqid=0 2024-11-11T20:41:45,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742381_1557 (size=12151) 2024-11-11T20:41:45,297 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/aa610fe7d41b40dcb3b325379b9abf14 2024-11-11T20:41:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/cdaff4cb82554d05bbfb7f457abe0225 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cdaff4cb82554d05bbfb7f457abe0225 2024-11-11T20:41:45,304 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cdaff4cb82554d05bbfb7f457abe0225, entries=150, sequenceid=245, filesize=11.9 K 2024-11-11T20:41:45,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7ab13ecf91de4c17b2d93b1db6328fed as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7ab13ecf91de4c17b2d93b1db6328fed 2024-11-11T20:41:45,309 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7ab13ecf91de4c17b2d93b1db6328fed, entries=150, sequenceid=245, filesize=11.9 K 2024-11-11T20:41:45,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/aa610fe7d41b40dcb3b325379b9abf14 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/aa610fe7d41b40dcb3b325379b9abf14 2024-11-11T20:41:45,312 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/aa610fe7d41b40dcb3b325379b9abf14, entries=150, sequenceid=245, filesize=11.9 K 2024-11-11T20:41:45,313 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1be5d7c75f632327a07f28f72dd94fc1 in 862ms, sequenceid=245, compaction requested=false 2024-11-11T20:41:45,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:45,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:45,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-11T20:41:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-11T20:41:45,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-11T20:41:45,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3200 sec 2024-11-11T20:41:45,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.3250 sec 2024-11-11T20:41:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:45,733 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:45,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T20:41:45,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/fe56f19882c44be39c69b8d6022cd643 is 50, key is test_row_0/A:col10/1731357704585/Put/seqid=0 2024-11-11T20:41:45,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742382_1558 (size=14741) 2024-11-11T20:41:45,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357765760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357765762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357765762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357765765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357765765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357765870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357765870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357765880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357765881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357766090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357766090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357766090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357766091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T20:41:46,141 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:46,142 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-11T20:41:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T20:41:46,143 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:46,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/fe56f19882c44be39c69b8d6022cd643 2024-11-11T20:41:46,144 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:46,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:46,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/175b053c9a51485084ae54ff9f1aeb2e is 50, key is test_row_0/B:col10/1731357704585/Put/seqid=0 2024-11-11T20:41:46,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742383_1559 (size=12301) 2024-11-11T20:41:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T20:41:46,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T20:41:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:46,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357766401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357766401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357766401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357766410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T20:41:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T20:41:46,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/175b053c9a51485084ae54ff9f1aeb2e 2024-11-11T20:41:46,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/91af99860e93464abc5248b5085fe9df is 50, key is test_row_0/C:col10/1731357704585/Put/seqid=0 2024-11-11T20:41:46,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742384_1560 (size=12301) 2024-11-11T20:41:46,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T20:41:46,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:46,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T20:41:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:46,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T20:41:46,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357766770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,782 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_2' in 1be5d7c75f632327a07f28f72dd94fc1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:41:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T20:41:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:46,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357766921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357766921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357766921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357766921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:46,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/91af99860e93464abc5248b5085fe9df 2024-11-11T20:41:46,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/fe56f19882c44be39c69b8d6022cd643 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fe56f19882c44be39c69b8d6022cd643 2024-11-11T20:41:46,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fe56f19882c44be39c69b8d6022cd643, entries=200, sequenceid=272, filesize=14.4 K 2024-11-11T20:41:46,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/175b053c9a51485084ae54ff9f1aeb2e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/175b053c9a51485084ae54ff9f1aeb2e 2024-11-11T20:41:46,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/175b053c9a51485084ae54ff9f1aeb2e, entries=150, sequenceid=272, filesize=12.0 K 2024-11-11T20:41:46,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/91af99860e93464abc5248b5085fe9df as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/91af99860e93464abc5248b5085fe9df 2024-11-11T20:41:46,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/91af99860e93464abc5248b5085fe9df, entries=150, sequenceid=272, filesize=12.0 K 2024-11-11T20:41:46,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1be5d7c75f632327a07f28f72dd94fc1 in 1248ms, sequenceid=272, compaction requested=true 2024-11-11T20:41:46,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:46,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:46,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:46,982 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:46,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:46,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:46,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:46,982 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:46,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:46,982 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39385 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:46,982 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/A is initiating minor compaction (all files) 2024-11-11T20:41:46,982 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:46,982 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/B is initiating minor compaction (all files) 2024-11-11T20:41:46,982 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/A in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,982 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/B in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:46,982 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/557b0c1f4d004f5aa07ff42a53bebf98, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cdaff4cb82554d05bbfb7f457abe0225, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fe56f19882c44be39c69b8d6022cd643] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=38.5 K 2024-11-11T20:41:46,982 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0629bfc86317426a9946d57395b007ff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7ab13ecf91de4c17b2d93b1db6328fed, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/175b053c9a51485084ae54ff9f1aeb2e] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=36.1 K 2024-11-11T20:41:46,983 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 557b0c1f4d004f5aa07ff42a53bebf98, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731357703561 2024-11-11T20:41:46,983 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0629bfc86317426a9946d57395b007ff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731357703561 2024-11-11T20:41:46,983 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdaff4cb82554d05bbfb7f457abe0225, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731357703911 2024-11-11T20:41:46,983 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ab13ecf91de4c17b2d93b1db6328fed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731357703911 2024-11-11T20:41:46,983 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe56f19882c44be39c69b8d6022cd643, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357704585 2024-11-11T20:41:46,983 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 175b053c9a51485084ae54ff9f1aeb2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357704585 2024-11-11T20:41:46,988 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:46,988 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#A#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:46,989 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/1c40b0fc323e4fac93bedb190e237606 is 50, key is test_row_0/B:col10/1731357704585/Put/seqid=0 2024-11-11T20:41:46,989 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/45f5ae9950804e3a95fd6e25b9b2f235 is 50, key is test_row_0/A:col10/1731357704585/Put/seqid=0 2024-11-11T20:41:46,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742385_1561 (size=12745) 2024-11-11T20:41:47,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742386_1562 (size=12745) 2024-11-11T20:41:47,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T20:41:47,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:47,059 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:41:47,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/1a4d2d01de3d451bbc0c6b7d94f0634c is 50, key is test_row_0/A:col10/1731357705761/Put/seqid=0 2024-11-11T20:41:47,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742387_1563 (size=12301) 2024-11-11T20:41:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T20:41:47,398 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/45f5ae9950804e3a95fd6e25b9b2f235 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45f5ae9950804e3a95fd6e25b9b2f235 2024-11-11T20:41:47,402 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/A of 1be5d7c75f632327a07f28f72dd94fc1 into 45f5ae9950804e3a95fd6e25b9b2f235(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:47,402 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:47,402 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/A, priority=13, startTime=1731357706981; duration=0sec 2024-11-11T20:41:47,402 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:47,402 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:A 2024-11-11T20:41:47,402 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:47,403 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:47,403 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/C is initiating minor compaction (all files) 2024-11-11T20:41:47,403 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/C in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:47,403 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/258bbf78c89f4e6ab04b4777b4db21bb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/aa610fe7d41b40dcb3b325379b9abf14, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/91af99860e93464abc5248b5085fe9df] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=36.2 K 2024-11-11T20:41:47,403 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 258bbf78c89f4e6ab04b4777b4db21bb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731357703561 2024-11-11T20:41:47,404 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa610fe7d41b40dcb3b325379b9abf14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731357703911 2024-11-11T20:41:47,404 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91af99860e93464abc5248b5085fe9df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357704585 2024-11-11T20:41:47,407 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/1c40b0fc323e4fac93bedb190e237606 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c40b0fc323e4fac93bedb190e237606 2024-11-11T20:41:47,410 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/B of 1be5d7c75f632327a07f28f72dd94fc1 into 1c40b0fc323e4fac93bedb190e237606(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:47,410 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:47,410 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/B, priority=13, startTime=1731357706982; duration=0sec 2024-11-11T20:41:47,410 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:47,410 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:B 2024-11-11T20:41:47,411 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:47,412 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/40c2145913374c8abc6367b1ca3f4942 is 50, key is test_row_0/C:col10/1731357704585/Put/seqid=0 2024-11-11T20:41:47,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742388_1564 (size=12915) 2024-11-11T20:41:47,465 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/1a4d2d01de3d451bbc0c6b7d94f0634c 2024-11-11T20:41:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/3dca793fd9334402b1688a1429f8bae4 is 50, key is test_row_0/B:col10/1731357705761/Put/seqid=0 2024-11-11T20:41:47,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742389_1565 (size=12301) 2024-11-11T20:41:47,474 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/3dca793fd9334402b1688a1429f8bae4 2024-11-11T20:41:47,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/c8aa6ccc848d48dea0f9abcb568e4f3a is 50, key is test_row_0/C:col10/1731357705761/Put/seqid=0 2024-11-11T20:41:47,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742390_1566 (size=12301) 2024-11-11T20:41:47,491 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/c8aa6ccc848d48dea0f9abcb568e4f3a 2024-11-11T20:41:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/1a4d2d01de3d451bbc0c6b7d94f0634c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/1a4d2d01de3d451bbc0c6b7d94f0634c 2024-11-11T20:41:47,500 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/1a4d2d01de3d451bbc0c6b7d94f0634c, entries=150, sequenceid=283, filesize=12.0 K 2024-11-11T20:41:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/3dca793fd9334402b1688a1429f8bae4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/3dca793fd9334402b1688a1429f8bae4 2024-11-11T20:41:47,504 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/3dca793fd9334402b1688a1429f8bae4, entries=150, sequenceid=283, filesize=12.0 K 2024-11-11T20:41:47,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/c8aa6ccc848d48dea0f9abcb568e4f3a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c8aa6ccc848d48dea0f9abcb568e4f3a 2024-11-11T20:41:47,508 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c8aa6ccc848d48dea0f9abcb568e4f3a, entries=150, sequenceid=283, filesize=12.0 K 2024-11-11T20:41:47,509 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 1be5d7c75f632327a07f28f72dd94fc1 in 450ms, sequenceid=283, compaction requested=false 2024-11-11T20:41:47,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:47,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:47,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-11T20:41:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-11T20:41:47,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-11T20:41:47,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3670 sec 2024-11-11T20:41:47,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 1.3710 sec 2024-11-11T20:41:47,822 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/40c2145913374c8abc6367b1ca3f4942 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/40c2145913374c8abc6367b1ca3f4942 2024-11-11T20:41:47,825 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/C of 1be5d7c75f632327a07f28f72dd94fc1 into 40c2145913374c8abc6367b1ca3f4942(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:47,826 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:47,826 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/C, priority=13, startTime=1731357706982; duration=0sec 2024-11-11T20:41:47,826 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:47,826 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:C 2024-11-11T20:41:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:47,944 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:47,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:41:47,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/f356bb38801143ecb136fd7fd934e274 is 50, key is test_row_0/A:col10/1731357707942/Put/seqid=0 2024-11-11T20:41:47,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742391_1567 (size=14741) 2024-11-11T20:41:47,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357767984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:47,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:47,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:47,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:47,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357767985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:47,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357767988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:47,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357767989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357768100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357768100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357768100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357768100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T20:41:48,291 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:48,291 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees 2024-11-11T20:41:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T20:41:48,293 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:48,294 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:48,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:48,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357768310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357768310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357768310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357768310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/f356bb38801143ecb136fd7fd934e274 2024-11-11T20:41:48,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/248aa9e7824a47399cb4b74f1dd630ef is 50, key is test_row_0/B:col10/1731357707942/Put/seqid=0 2024-11-11T20:41:48,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742392_1568 (size=12301) 2024-11-11T20:41:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T20:41:48,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-11T20:41:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:48,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-11T20:41:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:48,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T20:41:48,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357768620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357768620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357768621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357768621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-11T20:41:48,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:48,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:48,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:48,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/248aa9e7824a47399cb4b74f1dd630ef 2024-11-11T20:41:48,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/c05b3b70cb8f47aa98401b4d274d209c is 50, key is test_row_0/C:col10/1731357707942/Put/seqid=0 2024-11-11T20:41:48,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742393_1569 (size=12301) 2024-11-11T20:41:48,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357768792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:48,801 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_2' in 1be5d7c75f632327a07f28f72dd94fc1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:41:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:48,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-11T20:41:48,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:48,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:48,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:48,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T20:41:49,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-11T20:41:49,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:49,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:49,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:49,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:49,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:49,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357769140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:49,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357769140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:49,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357769141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:49,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357769141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:49,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/c05b3b70cb8f47aa98401b4d274d209c 2024-11-11T20:41:49,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/f356bb38801143ecb136fd7fd934e274 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/f356bb38801143ecb136fd7fd934e274 2024-11-11T20:41:49,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/f356bb38801143ecb136fd7fd934e274, entries=200, sequenceid=298, filesize=14.4 K 2024-11-11T20:41:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/248aa9e7824a47399cb4b74f1dd630ef as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/248aa9e7824a47399cb4b74f1dd630ef 2024-11-11T20:41:49,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/248aa9e7824a47399cb4b74f1dd630ef, entries=150, sequenceid=298, filesize=12.0 K 2024-11-11T20:41:49,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/c05b3b70cb8f47aa98401b4d274d209c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c05b3b70cb8f47aa98401b4d274d209c 2024-11-11T20:41:49,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c05b3b70cb8f47aa98401b4d274d209c, entries=150, sequenceid=298, filesize=12.0 K 2024-11-11T20:41:49,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1be5d7c75f632327a07f28f72dd94fc1 in 1245ms, sequenceid=298, compaction requested=true 2024-11-11T20:41:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:49,189 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:49,189 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:49,189 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:49,189 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:49,189 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/B is initiating minor compaction (all files) 2024-11-11T20:41:49,189 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/A is initiating minor compaction (all files) 2024-11-11T20:41:49,190 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/B in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:49,190 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/A in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:49,190 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45f5ae9950804e3a95fd6e25b9b2f235, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/1a4d2d01de3d451bbc0c6b7d94f0634c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/f356bb38801143ecb136fd7fd934e274] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=38.9 K 2024-11-11T20:41:49,190 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c40b0fc323e4fac93bedb190e237606, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/3dca793fd9334402b1688a1429f8bae4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/248aa9e7824a47399cb4b74f1dd630ef] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=36.5 K 2024-11-11T20:41:49,190 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 45f5ae9950804e3a95fd6e25b9b2f235, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357704585 2024-11-11T20:41:49,190 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c40b0fc323e4fac93bedb190e237606, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357704585 2024-11-11T20:41:49,190 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a4d2d01de3d451bbc0c6b7d94f0634c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731357705761 2024-11-11T20:41:49,190 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dca793fd9334402b1688a1429f8bae4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731357705761 2024-11-11T20:41:49,190 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f356bb38801143ecb136fd7fd934e274, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731357707939 2024-11-11T20:41:49,190 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 248aa9e7824a47399cb4b74f1dd630ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731357707939 2024-11-11T20:41:49,195 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#A#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:49,195 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:49,196 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/27577745c8cf42c1ab828b45861efcac is 50, key is test_row_0/A:col10/1731357707942/Put/seqid=0 2024-11-11T20:41:49,196 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/f972b5fa15dc4fe49eb4f6e89f144a1d is 50, key is test_row_0/B:col10/1731357707942/Put/seqid=0 2024-11-11T20:41:49,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742395_1571 (size=12847) 2024-11-11T20:41:49,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742394_1570 (size=12847) 2024-11-11T20:41:49,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-11T20:41:49,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:49,211 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:41:49,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/7ebd753a8db0409f8f3575b88eab858d is 50, key is test_row_0/A:col10/1731357707971/Put/seqid=0 2024-11-11T20:41:49,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742396_1572 (size=12301) 2024-11-11T20:41:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T20:41:49,602 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/27577745c8cf42c1ab828b45861efcac as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/27577745c8cf42c1ab828b45861efcac 2024-11-11T20:41:49,602 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/f972b5fa15dc4fe49eb4f6e89f144a1d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f972b5fa15dc4fe49eb4f6e89f144a1d 2024-11-11T20:41:49,605 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/B of 1be5d7c75f632327a07f28f72dd94fc1 into f972b5fa15dc4fe49eb4f6e89f144a1d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:49,605 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:49,605 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/B, priority=13, startTime=1731357709189; duration=0sec 2024-11-11T20:41:49,605 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:49,605 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:B 2024-11-11T20:41:49,605 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:49,606 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/A of 1be5d7c75f632327a07f28f72dd94fc1 into 27577745c8cf42c1ab828b45861efcac(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:49,606 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:49,606 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/A, priority=13, startTime=1731357709189; duration=0sec 2024-11-11T20:41:49,606 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:49,606 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:49,606 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:A 2024-11-11T20:41:49,606 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/C is initiating minor compaction (all files) 2024-11-11T20:41:49,606 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/C in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:49,606 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/40c2145913374c8abc6367b1ca3f4942, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c8aa6ccc848d48dea0f9abcb568e4f3a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c05b3b70cb8f47aa98401b4d274d209c] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=36.6 K 2024-11-11T20:41:49,606 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40c2145913374c8abc6367b1ca3f4942, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731357704585 2024-11-11T20:41:49,607 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8aa6ccc848d48dea0f9abcb568e4f3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731357705761 2024-11-11T20:41:49,607 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c05b3b70cb8f47aa98401b4d274d209c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731357707939 2024-11-11T20:41:49,612 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:49,612 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/b25a26ec5a4c47cfa94dc49d008d4c5a is 50, key is test_row_0/C:col10/1731357707942/Put/seqid=0 2024-11-11T20:41:49,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742397_1573 (size=13017) 2024-11-11T20:41:49,618 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/7ebd753a8db0409f8f3575b88eab858d 2024-11-11T20:41:49,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/44b159a7b3fc44a197d41b3c12aeb130 is 50, key is test_row_0/B:col10/1731357707971/Put/seqid=0 2024-11-11T20:41:49,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742398_1574 (size=12301) 2024-11-11T20:41:49,627 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/44b159a7b3fc44a197d41b3c12aeb130 2024-11-11T20:41:49,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/4e1e67f959a346499a05e20933aac699 is 50, key is test_row_0/C:col10/1731357707971/Put/seqid=0 2024-11-11T20:41:49,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742399_1575 (size=12301) 2024-11-11T20:41:50,018 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/b25a26ec5a4c47cfa94dc49d008d4c5a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b25a26ec5a4c47cfa94dc49d008d4c5a 2024-11-11T20:41:50,021 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/C of 1be5d7c75f632327a07f28f72dd94fc1 into b25a26ec5a4c47cfa94dc49d008d4c5a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:50,021 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:50,021 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/C, priority=13, startTime=1731357709189; duration=0sec 2024-11-11T20:41:50,021 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:50,021 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:C 2024-11-11T20:41:50,040 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/4e1e67f959a346499a05e20933aac699 2024-11-11T20:41:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/7ebd753a8db0409f8f3575b88eab858d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ebd753a8db0409f8f3575b88eab858d 2024-11-11T20:41:50,045 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ebd753a8db0409f8f3575b88eab858d, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T20:41:50,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/44b159a7b3fc44a197d41b3c12aeb130 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/44b159a7b3fc44a197d41b3c12aeb130 2024-11-11T20:41:50,048 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/44b159a7b3fc44a197d41b3c12aeb130, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T20:41:50,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/4e1e67f959a346499a05e20933aac699 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4e1e67f959a346499a05e20933aac699 2024-11-11T20:41:50,051 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4e1e67f959a346499a05e20933aac699, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T20:41:50,052 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 1be5d7c75f632327a07f28f72dd94fc1 in 841ms, sequenceid=322, compaction requested=false 2024-11-11T20:41:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=148 2024-11-11T20:41:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=148 2024-11-11T20:41:50,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-11T20:41:50,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7670 sec 2024-11-11T20:41:50,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees in 1.7710 sec 2024-11-11T20:41:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:50,165 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:50,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:41:50,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/db26b200302e4e39951d3b5a04e62072 is 50, key is test_row_0/A:col10/1731357710165/Put/seqid=0 2024-11-11T20:41:50,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742400_1576 (size=14741) 2024-11-11T20:41:50,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/db26b200302e4e39951d3b5a04e62072 2024-11-11T20:41:50,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7e61f0a502e04e9aa3a44637682e9c9d is 50, key is test_row_0/B:col10/1731357710165/Put/seqid=0 2024-11-11T20:41:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742401_1577 (size=12301) 2024-11-11T20:41:50,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357770242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357770242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357770243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357770256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357770360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357770360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357770360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357770371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T20:41:50,441 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:50,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees 2024-11-11T20:41:50,442 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-11T20:41:50,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:50,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-11T20:41:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357770570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357770571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357770571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7e61f0a502e04e9aa3a44637682e9c9d 2024-11-11T20:41:50,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357770581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/fa6ec5df972543ef99a2d76e170e039c is 50, key is test_row_0/C:col10/1731357710165/Put/seqid=0 2024-11-11T20:41:50,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742402_1578 (size=12301) 2024-11-11T20:41:50,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-11T20:41:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:50,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:50,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-11T20:41:50,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:50,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:50,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:50,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:50,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-11T20:41:50,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357770880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357770890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357770891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-11T20:41:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:50,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357770901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:50,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/fa6ec5df972543ef99a2d76e170e039c 2024-11-11T20:41:50,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/db26b200302e4e39951d3b5a04e62072 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/db26b200302e4e39951d3b5a04e62072 2024-11-11T20:41:50,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/db26b200302e4e39951d3b5a04e62072, entries=200, sequenceid=337, filesize=14.4 K 2024-11-11T20:41:50,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/7e61f0a502e04e9aa3a44637682e9c9d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e61f0a502e04e9aa3a44637682e9c9d 2024-11-11T20:41:51,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e61f0a502e04e9aa3a44637682e9c9d, entries=150, sequenceid=337, filesize=12.0 K 2024-11-11T20:41:51,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/fa6ec5df972543ef99a2d76e170e039c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/fa6ec5df972543ef99a2d76e170e039c 2024-11-11T20:41:51,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/fa6ec5df972543ef99a2d76e170e039c, entries=150, sequenceid=337, filesize=12.0 K 2024-11-11T20:41:51,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1be5d7c75f632327a07f28f72dd94fc1 in 841ms, sequenceid=337, compaction requested=true 2024-11-11T20:41:51,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:51,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:51,006 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:51,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:51,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:51,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:51,006 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:51,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:51,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:51,007 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:51,007 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/A is initiating minor compaction (all files) 2024-11-11T20:41:51,007 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:51,007 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/A in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:51,007 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/B is initiating minor compaction (all files) 2024-11-11T20:41:51,007 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/B in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:51,007 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/27577745c8cf42c1ab828b45861efcac, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ebd753a8db0409f8f3575b88eab858d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/db26b200302e4e39951d3b5a04e62072] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=39.0 K 2024-11-11T20:41:51,007 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f972b5fa15dc4fe49eb4f6e89f144a1d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/44b159a7b3fc44a197d41b3c12aeb130, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e61f0a502e04e9aa3a44637682e9c9d] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=36.6 K 2024-11-11T20:41:51,007 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27577745c8cf42c1ab828b45861efcac, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731357707939 2024-11-11T20:41:51,007 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f972b5fa15dc4fe49eb4f6e89f144a1d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731357707939 2024-11-11T20:41:51,008 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ebd753a8db0409f8f3575b88eab858d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731357707971 2024-11-11T20:41:51,008 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 44b159a7b3fc44a197d41b3c12aeb130, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731357707971 2024-11-11T20:41:51,008 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e61f0a502e04e9aa3a44637682e9c9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731357710164 2024-11-11T20:41:51,008 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting db26b200302e4e39951d3b5a04e62072, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731357710164 2024-11-11T20:41:51,016 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:51,017 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/0bf4fac2d3ee4499af01d8c94290668c is 50, key is test_row_0/B:col10/1731357710165/Put/seqid=0 2024-11-11T20:41:51,019 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#A#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:51,019 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/75bc1a6235d349e1b492e6adf70d29ad is 50, key is test_row_0/A:col10/1731357710165/Put/seqid=0 2024-11-11T20:41:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742403_1579 (size=12949) 2024-11-11T20:41:51,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742404_1580 (size=12949) 2024-11-11T20:41:51,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-11T20:41:51,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:51,052 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:41:51,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/fc07b0720a8c4557861c4d9b764d785b is 50, key is test_row_0/A:col10/1731357710226/Put/seqid=0 2024-11-11T20:41:51,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742405_1581 (size=12301) 2024-11-11T20:41:51,065 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/fc07b0720a8c4557861c4d9b764d785b 2024-11-11T20:41:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-11T20:41:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/12c00e506179479e80839e0a9a34a5b2 is 50, key is test_row_0/B:col10/1731357710226/Put/seqid=0 2024-11-11T20:41:51,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742406_1582 (size=12301) 2024-11-11T20:41:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:51,431 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/0bf4fac2d3ee4499af01d8c94290668c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0bf4fac2d3ee4499af01d8c94290668c 2024-11-11T20:41:51,433 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/75bc1a6235d349e1b492e6adf70d29ad as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/75bc1a6235d349e1b492e6adf70d29ad 2024-11-11T20:41:51,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357771425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357771425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,436 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/B of 1be5d7c75f632327a07f28f72dd94fc1 into 0bf4fac2d3ee4499af01d8c94290668c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:51,436 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:51,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,436 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/B, priority=13, startTime=1731357711006; duration=0sec 2024-11-11T20:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357771426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:51,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:B 2024-11-11T20:41:51,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,437 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/A of 1be5d7c75f632327a07f28f72dd94fc1 into 75bc1a6235d349e1b492e6adf70d29ad(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:51,437 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:51,437 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/A, priority=13, startTime=1731357711006; duration=0sec 2024-11-11T20:41:51,437 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:51,437 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:A 2024-11-11T20:41:51,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:51,437 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/C is initiating minor compaction (all files) 2024-11-11T20:41:51,438 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/C in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:51,438 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b25a26ec5a4c47cfa94dc49d008d4c5a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4e1e67f959a346499a05e20933aac699, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/fa6ec5df972543ef99a2d76e170e039c] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=36.7 K 2024-11-11T20:41:51,438 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b25a26ec5a4c47cfa94dc49d008d4c5a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731357707939 2024-11-11T20:41:51,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357771435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,438 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1e67f959a346499a05e20933aac699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731357707971 2024-11-11T20:41:51,438 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting fa6ec5df972543ef99a2d76e170e039c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731357710164 2024-11-11T20:41:51,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,443 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#C#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:51,443 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/1b65443e2b6d46c5af2b5f0b04c32492 is 50, key is test_row_0/C:col10/1731357710165/Put/seqid=0 2024-11-11T20:41:51,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742407_1583 (size=13119) 2024-11-11T20:41:51,479 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/12c00e506179479e80839e0a9a34a5b2 2024-11-11T20:41:51,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/cb1989cfaf644d7a90f9a89c6ad77743 is 50, key is test_row_0/C:col10/1731357710226/Put/seqid=0 2024-11-11T20:41:51,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742408_1584 (size=12301) 2024-11-11T20:41:51,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357771540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357771540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357771540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357771541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-11T20:41:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357771750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357771750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357771751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357771751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:51,849 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/1b65443e2b6d46c5af2b5f0b04c32492 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1b65443e2b6d46c5af2b5f0b04c32492 2024-11-11T20:41:51,854 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/C of 1be5d7c75f632327a07f28f72dd94fc1 into 1b65443e2b6d46c5af2b5f0b04c32492(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:51,854 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:51,854 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/C, priority=13, startTime=1731357711006; duration=0sec 2024-11-11T20:41:51,854 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:51,854 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:C 2024-11-11T20:41:51,894 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/cb1989cfaf644d7a90f9a89c6ad77743 2024-11-11T20:41:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/fc07b0720a8c4557861c4d9b764d785b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fc07b0720a8c4557861c4d9b764d785b 2024-11-11T20:41:51,900 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fc07b0720a8c4557861c4d9b764d785b, entries=150, sequenceid=361, filesize=12.0 K 2024-11-11T20:41:51,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/12c00e506179479e80839e0a9a34a5b2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/12c00e506179479e80839e0a9a34a5b2 2024-11-11T20:41:51,903 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/12c00e506179479e80839e0a9a34a5b2, entries=150, sequenceid=361, filesize=12.0 K 2024-11-11T20:41:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/cb1989cfaf644d7a90f9a89c6ad77743 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cb1989cfaf644d7a90f9a89c6ad77743 2024-11-11T20:41:51,907 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cb1989cfaf644d7a90f9a89c6ad77743, entries=150, sequenceid=361, filesize=12.0 K 2024-11-11T20:41:51,907 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1be5d7c75f632327a07f28f72dd94fc1 in 855ms, sequenceid=361, compaction requested=false 2024-11-11T20:41:51,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:51,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:51,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150 2024-11-11T20:41:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=150 2024-11-11T20:41:51,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-11T20:41:51,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-11-11T20:41:51,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees in 1.4690 sec 2024-11-11T20:41:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:52,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T20:41:52,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/465b984a9d65473bbc9076b6a7b8fefd is 50, key is test_row_0/A:col10/1731357712063/Put/seqid=0 2024-11-11T20:41:52,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742409_1585 (size=14741) 2024-11-11T20:41:52,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/465b984a9d65473bbc9076b6a7b8fefd 2024-11-11T20:41:52,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/1c4a78831bbf4c4e8c0f1d8960c442f4 is 50, key is test_row_0/B:col10/1731357712063/Put/seqid=0 2024-11-11T20:41:52,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742410_1586 (size=12301) 2024-11-11T20:41:52,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/1c4a78831bbf4c4e8c0f1d8960c442f4 2024-11-11T20:41:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357772124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/003dcadd078f40f28d3445f77c4a04c5 is 50, key is test_row_0/C:col10/1731357712063/Put/seqid=0 2024-11-11T20:41:52,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357772125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357772126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357772128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742411_1587 (size=12301) 2024-11-11T20:41:52,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/003dcadd078f40f28d3445f77c4a04c5 2024-11-11T20:41:52,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/465b984a9d65473bbc9076b6a7b8fefd as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/465b984a9d65473bbc9076b6a7b8fefd 2024-11-11T20:41:52,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/465b984a9d65473bbc9076b6a7b8fefd, entries=200, sequenceid=377, filesize=14.4 K 2024-11-11T20:41:52,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/1c4a78831bbf4c4e8c0f1d8960c442f4 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c4a78831bbf4c4e8c0f1d8960c442f4 2024-11-11T20:41:52,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c4a78831bbf4c4e8c0f1d8960c442f4, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T20:41:52,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/003dcadd078f40f28d3445f77c4a04c5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/003dcadd078f40f28d3445f77c4a04c5 2024-11-11T20:41:52,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/003dcadd078f40f28d3445f77c4a04c5, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T20:41:52,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1be5d7c75f632327a07f28f72dd94fc1 in 90ms, sequenceid=377, compaction requested=true 2024-11-11T20:41:52,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:52,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:52,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:52,154 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:52,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:52,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:52,154 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:52,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:52,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:52,157 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:52,157 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/B is initiating minor compaction (all files) 2024-11-11T20:41:52,157 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:52,157 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/B in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:52,157 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/A is initiating minor compaction (all files) 2024-11-11T20:41:52,157 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/A in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:52,157 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0bf4fac2d3ee4499af01d8c94290668c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/12c00e506179479e80839e0a9a34a5b2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c4a78831bbf4c4e8c0f1d8960c442f4] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=36.7 K 2024-11-11T20:41:52,157 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/75bc1a6235d349e1b492e6adf70d29ad, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fc07b0720a8c4557861c4d9b764d785b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/465b984a9d65473bbc9076b6a7b8fefd] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=39.1 K 2024-11-11T20:41:52,158 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75bc1a6235d349e1b492e6adf70d29ad, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731357710164 2024-11-11T20:41:52,158 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc07b0720a8c4557861c4d9b764d785b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731357710212 2024-11-11T20:41:52,158 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 465b984a9d65473bbc9076b6a7b8fefd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731357711424 2024-11-11T20:41:52,160 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bf4fac2d3ee4499af01d8c94290668c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731357710164 2024-11-11T20:41:52,160 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 12c00e506179479e80839e0a9a34a5b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731357710212 2024-11-11T20:41:52,160 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c4a78831bbf4c4e8c0f1d8960c442f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731357711425 2024-11-11T20:41:52,164 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#A#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:52,164 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/86f145087acc4f87a8c6f6a5324c0f54 is 50, key is test_row_0/A:col10/1731357712063/Put/seqid=0 2024-11-11T20:41:52,174 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#B#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:52,174 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/fe5ce28883e04fa8a67b84ca4ddd24d7 is 50, key is test_row_0/B:col10/1731357712063/Put/seqid=0 2024-11-11T20:41:52,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742412_1588 (size=13051) 2024-11-11T20:41:52,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742413_1589 (size=13051) 2024-11-11T20:41:52,189 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/86f145087acc4f87a8c6f6a5324c0f54 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/86f145087acc4f87a8c6f6a5324c0f54 2024-11-11T20:41:52,189 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/fe5ce28883e04fa8a67b84ca4ddd24d7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe5ce28883e04fa8a67b84ca4ddd24d7 2024-11-11T20:41:52,196 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/A of 1be5d7c75f632327a07f28f72dd94fc1 into 86f145087acc4f87a8c6f6a5324c0f54(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:52,196 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/B of 1be5d7c75f632327a07f28f72dd94fc1 into fe5ce28883e04fa8a67b84ca4ddd24d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:52,196 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:52,196 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:52,196 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/A, priority=13, startTime=1731357712154; duration=0sec 2024-11-11T20:41:52,196 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/B, priority=13, startTime=1731357712154; duration=0sec 2024-11-11T20:41:52,196 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:52,196 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:A 2024-11-11T20:41:52,196 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:52,196 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:B 2024-11-11T20:41:52,196 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:41:52,198 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:41:52,198 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/C is initiating minor compaction (all files) 2024-11-11T20:41:52,199 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/C in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:52,199 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1b65443e2b6d46c5af2b5f0b04c32492, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cb1989cfaf644d7a90f9a89c6ad77743, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/003dcadd078f40f28d3445f77c4a04c5] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=36.8 K 2024-11-11T20:41:52,199 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b65443e2b6d46c5af2b5f0b04c32492, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731357710164 2024-11-11T20:41:52,199 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1989cfaf644d7a90f9a89c6ad77743, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731357710212 2024-11-11T20:41:52,199 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 003dcadd078f40f28d3445f77c4a04c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731357711425 2024-11-11T20:41:52,203 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:52,204 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/95b0b27769d04c00ac18991778035d89 is 50, key is test_row_0/C:col10/1731357712063/Put/seqid=0 2024-11-11T20:41:52,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742414_1590 (size=13221) 2024-11-11T20:41:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:52,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T20:41:52,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/96ecfb6bceab406bbd5225969ec52f0b is 50, key is test_row_0/A:col10/1731357712123/Put/seqid=0 2024-11-11T20:41:52,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742415_1591 (size=12301) 2024-11-11T20:41:52,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357772259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357772259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357772263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357772267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357772370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357772381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357772381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357772381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357772580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-11T20:41:52,591 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:41:52,591 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:41:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees 2024-11-11T20:41:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-11T20:41:52,593 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:41:52,593 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:41:52,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:41:52,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357772591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357772591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357772591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,618 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/95b0b27769d04c00ac18991778035d89 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/95b0b27769d04c00ac18991778035d89 2024-11-11T20:41:52,621 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/C of 1be5d7c75f632327a07f28f72dd94fc1 into 95b0b27769d04c00ac18991778035d89(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:52,621 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:52,621 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/C, priority=13, startTime=1731357712154; duration=0sec 2024-11-11T20:41:52,621 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:52,621 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:C 2024-11-11T20:41:52,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/96ecfb6bceab406bbd5225969ec52f0b 2024-11-11T20:41:52,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/83b456d163134c54a93d5b76e00dc59e is 50, key is test_row_0/B:col10/1731357712123/Put/seqid=0 2024-11-11T20:41:52,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742416_1592 (size=12301) 2024-11-11T20:41:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-11T20:41:52,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-11T20:41:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:52,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:52,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:52,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:52,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51090 deadline: 1731357772821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,825 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_2' in 1be5d7c75f632327a07f28f72dd94fc1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:41:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-11T20:41:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:52,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357772901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-11T20:41:52,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357772911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357772912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357772912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:53,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-11T20:41:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:53,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:53,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/83b456d163134c54a93d5b76e00dc59e 2024-11-11T20:41:53,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/ec2673d47f34421fa7505f33b858b45d is 50, key is test_row_0/C:col10/1731357712123/Put/seqid=0 2024-11-11T20:41:53,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742417_1593 (size=12301) 2024-11-11T20:41:53,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-11T20:41:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:53,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:53,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-11T20:41:53,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-11T20:41:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:53,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:41:53,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357773410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:53,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357773420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:53,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357773421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357773431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:53,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/ec2673d47f34421fa7505f33b858b45d 2024-11-11T20:41:53,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/96ecfb6bceab406bbd5225969ec52f0b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/96ecfb6bceab406bbd5225969ec52f0b 2024-11-11T20:41:53,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/96ecfb6bceab406bbd5225969ec52f0b, entries=150, sequenceid=405, filesize=12.0 K 2024-11-11T20:41:53,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/83b456d163134c54a93d5b76e00dc59e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/83b456d163134c54a93d5b76e00dc59e 2024-11-11T20:41:53,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/83b456d163134c54a93d5b76e00dc59e, entries=150, sequenceid=405, filesize=12.0 K 2024-11-11T20:41:53,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/ec2673d47f34421fa7505f33b858b45d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/ec2673d47f34421fa7505f33b858b45d 2024-11-11T20:41:53,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/ec2673d47f34421fa7505f33b858b45d, entries=150, sequenceid=405, filesize=12.0 K 2024-11-11T20:41:53,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1be5d7c75f632327a07f28f72dd94fc1 in 1249ms, sequenceid=405, compaction requested=false 2024-11-11T20:41:53,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:53,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-11T20:41:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:53,509 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T20:41:53,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/302aa3b7dac7491e86597ccc976d1576 is 50, key is test_row_0/A:col10/1731357712262/Put/seqid=0 2024-11-11T20:41:53,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742418_1594 (size=12301) 2024-11-11T20:41:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-11T20:41:53,916 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/302aa3b7dac7491e86597ccc976d1576 2024-11-11T20:41:53,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/009fd171a0f14426a757539926911fdb is 50, key is test_row_0/B:col10/1731357712262/Put/seqid=0 2024-11-11T20:41:53,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742419_1595 (size=12301) 2024-11-11T20:41:54,326 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/009fd171a0f14426a757539926911fdb 2024-11-11T20:41:54,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/b327c006a7e844189fc734dee1544d92 is 50, key is test_row_0/C:col10/1731357712262/Put/seqid=0 2024-11-11T20:41:54,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742420_1596 (size=12301) 2024-11-11T20:41:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. as already flushing 2024-11-11T20:41:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:54,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357774483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357774491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357774492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357774493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357774601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357774601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357774611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357774611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,736 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/b327c006a7e844189fc734dee1544d92 2024-11-11T20:41:54,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/302aa3b7dac7491e86597ccc976d1576 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/302aa3b7dac7491e86597ccc976d1576 2024-11-11T20:41:54,745 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/302aa3b7dac7491e86597ccc976d1576, entries=150, sequenceid=416, filesize=12.0 K 2024-11-11T20:41:54,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/009fd171a0f14426a757539926911fdb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/009fd171a0f14426a757539926911fdb 2024-11-11T20:41:54,748 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/009fd171a0f14426a757539926911fdb, entries=150, sequenceid=416, filesize=12.0 K 2024-11-11T20:41:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/b327c006a7e844189fc734dee1544d92 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b327c006a7e844189fc734dee1544d92 2024-11-11T20:41:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-11T20:41:54,753 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b327c006a7e844189fc734dee1544d92, entries=150, sequenceid=416, filesize=12.0 K 2024-11-11T20:41:54,754 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 1be5d7c75f632327a07f28f72dd94fc1 in 1245ms, sequenceid=416, compaction requested=true 2024-11-11T20:41:54,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:54,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:54,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-11-11T20:41:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=152 2024-11-11T20:41:54,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-11T20:41:54,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1620 sec 2024-11-11T20:41:54,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees in 2.1670 sec 2024-11-11T20:41:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:54,819 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:54,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-11T20:41:54,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/632c281770504b60b7d52af1e71f1e85 is 50, key is test_row_0/A:col10/1731357714491/Put/seqid=0 2024-11-11T20:41:54,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742421_1597 (size=14741) 2024-11-11T20:41:54,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/632c281770504b60b7d52af1e71f1e85 2024-11-11T20:41:54,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/136f8041f1594168b284567b0a289a15 is 50, key is test_row_0/B:col10/1731357714491/Put/seqid=0 2024-11-11T20:41:54,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742422_1598 (size=12301) 2024-11-11T20:41:54,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/136f8041f1594168b284567b0a289a15 2024-11-11T20:41:54,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357774837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/4c038ef69af84f7aae18ee5eaa60cdac is 50, key is test_row_0/C:col10/1731357714491/Put/seqid=0 2024-11-11T20:41:54,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357774840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357774842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742423_1599 (size=12301) 2024-11-11T20:41:54,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357774843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357774950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357774950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357774951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:54,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357774961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:55,141 INFO [Thread-2257 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2257. 2024-11-11T20:41:55,141 DEBUG [Thread-2257 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:55,141 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:55,141 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,142 INFO [Thread-2255 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2255. 2024-11-11T20:41:55,142 DEBUG [Thread-2255 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:55,142 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,142 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,142 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:55,145 INFO [Thread-2259 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2259. 2024-11-11T20:41:55,145 DEBUG [Thread-2259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:55,145 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,145 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:55,150 INFO [Thread-2253 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2253. 2024-11-11T20:41:55,150 DEBUG [Thread-2253 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:55,150 DEBUG [Thread-2253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,150 DEBUG [Thread-2253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:55,150 INFO [Thread-2251 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2251. 2024-11-11T20:41:55,150 DEBUG [Thread-2251 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:55,150 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,151 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:55,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:55,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:55,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51004 deadline: 1731357775161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51084 deadline: 1731357775161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51046 deadline: 1731357775161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:55,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:41:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1731357775181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:41:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 , the old value is region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:41:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:41:55,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/4c038ef69af84f7aae18ee5eaa60cdac 2024-11-11T20:41:55,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/632c281770504b60b7d52af1e71f1e85 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/632c281770504b60b7d52af1e71f1e85 2024-11-11T20:41:55,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/632c281770504b60b7d52af1e71f1e85, entries=200, sequenceid=444, filesize=14.4 K 2024-11-11T20:41:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/136f8041f1594168b284567b0a289a15 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/136f8041f1594168b284567b0a289a15 2024-11-11T20:41:55,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/136f8041f1594168b284567b0a289a15, entries=150, sequenceid=444, filesize=12.0 K 2024-11-11T20:41:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/4c038ef69af84f7aae18ee5eaa60cdac as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4c038ef69af84f7aae18ee5eaa60cdac 2024-11-11T20:41:55,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4c038ef69af84f7aae18ee5eaa60cdac, entries=150, sequenceid=444, filesize=12.0 K 2024-11-11T20:41:55,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 1be5d7c75f632327a07f28f72dd94fc1 in 443ms, sequenceid=444, compaction requested=true 2024-11-11T20:41:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:41:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:41:55,262 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1be5d7c75f632327a07f28f72dd94fc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:41:55,262 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:55,263 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:55,263 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:55,263 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/A is initiating minor compaction (all files) 2024-11-11T20:41:55,263 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/B is initiating minor compaction (all files) 2024-11-11T20:41:55,263 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/B in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:55,263 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/A in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:55,263 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe5ce28883e04fa8a67b84ca4ddd24d7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/83b456d163134c54a93d5b76e00dc59e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/009fd171a0f14426a757539926911fdb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/136f8041f1594168b284567b0a289a15] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=48.8 K 2024-11-11T20:41:55,263 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/86f145087acc4f87a8c6f6a5324c0f54, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/96ecfb6bceab406bbd5225969ec52f0b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/302aa3b7dac7491e86597ccc976d1576, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/632c281770504b60b7d52af1e71f1e85] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=51.2 K 2024-11-11T20:41:55,263 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f145087acc4f87a8c6f6a5324c0f54, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731357711425 2024-11-11T20:41:55,263 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe5ce28883e04fa8a67b84ca4ddd24d7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731357711425 2024-11-11T20:41:55,264 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83b456d163134c54a93d5b76e00dc59e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731357712123 2024-11-11T20:41:55,264 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ecfb6bceab406bbd5225969ec52f0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731357712123 2024-11-11T20:41:55,264 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 302aa3b7dac7491e86597ccc976d1576, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731357712256 2024-11-11T20:41:55,264 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 009fd171a0f14426a757539926911fdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731357712256 2024-11-11T20:41:55,264 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 136f8041f1594168b284567b0a289a15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731357714481 2024-11-11T20:41:55,264 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 632c281770504b60b7d52af1e71f1e85, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731357714477 2024-11-11T20:41:55,270 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#A#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:55,271 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/a9fbeb0e6e9245a29ad9bae0473c14a8 is 50, key is test_row_0/A:col10/1731357714491/Put/seqid=0 2024-11-11T20:41:55,271 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#B#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:55,271 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/2ca20b28712040dab51a103fffdd4ed6 is 50, key is test_row_0/B:col10/1731357714491/Put/seqid=0 2024-11-11T20:41:55,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742424_1600 (size=13187) 2024-11-11T20:41:55,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742425_1601 (size=13187) 2024-11-11T20:41:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:41:55,472 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:41:55,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:41:55,472 INFO [Thread-2248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2248. 2024-11-11T20:41:55,472 INFO [Thread-2242 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2242. 2024-11-11T20:41:55,472 DEBUG [Thread-2248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:55,472 INFO [Thread-2244 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2244. 2024-11-11T20:41:55,472 DEBUG [Thread-2242 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:55,472 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,472 DEBUG [Thread-2242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,472 DEBUG [Thread-2244 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:55,472 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,472 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,472 DEBUG [Thread-2242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,472 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:55,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:55,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:55,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/00d5abdc8e5f43ffa81f1ecaa5297a78 is 50, key is test_row_0/A:col10/1731357715471/Put/seqid=0 2024-11-11T20:41:55,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742426_1602 (size=12301) 2024-11-11T20:41:55,491 INFO [Thread-2240 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2240. 2024-11-11T20:41:55,491 DEBUG [Thread-2240 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:41:55,491 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,492 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:41:55,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:41:55,680 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/a9fbeb0e6e9245a29ad9bae0473c14a8 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/a9fbeb0e6e9245a29ad9bae0473c14a8 2024-11-11T20:41:55,682 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/2ca20b28712040dab51a103fffdd4ed6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/2ca20b28712040dab51a103fffdd4ed6 2024-11-11T20:41:55,684 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/A of 1be5d7c75f632327a07f28f72dd94fc1 into a9fbeb0e6e9245a29ad9bae0473c14a8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:55,684 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:55,684 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/A, priority=12, startTime=1731357715262; duration=0sec 2024-11-11T20:41:55,684 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:41:55,684 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:A 2024-11-11T20:41:55,684 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:41:55,685 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:41:55,685 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): 1be5d7c75f632327a07f28f72dd94fc1/C is initiating minor compaction (all files) 2024-11-11T20:41:55,685 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1be5d7c75f632327a07f28f72dd94fc1/C in TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:41:55,685 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/95b0b27769d04c00ac18991778035d89, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/ec2673d47f34421fa7505f33b858b45d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b327c006a7e844189fc734dee1544d92, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4c038ef69af84f7aae18ee5eaa60cdac] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp, totalSize=48.9 K 2024-11-11T20:41:55,685 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 95b0b27769d04c00ac18991778035d89, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731357711425 2024-11-11T20:41:55,686 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2673d47f34421fa7505f33b858b45d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731357712123 2024-11-11T20:41:55,686 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b327c006a7e844189fc734dee1544d92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731357712256 2024-11-11T20:41:55,686 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/B of 1be5d7c75f632327a07f28f72dd94fc1 into 2ca20b28712040dab51a103fffdd4ed6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:55,686 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:55,686 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/B, priority=12, startTime=1731357715262; duration=0sec 2024-11-11T20:41:55,686 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c038ef69af84f7aae18ee5eaa60cdac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731357714481 2024-11-11T20:41:55,686 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:55,686 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:B 2024-11-11T20:41:55,693 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1be5d7c75f632327a07f28f72dd94fc1#C#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:41:55,693 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/2ba4cf0c5d914253a1af5ed1b5c47f2a is 50, key is test_row_0/C:col10/1731357714491/Put/seqid=0 2024-11-11T20:41:55,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742427_1603 (size=13357) 2024-11-11T20:41:55,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/00d5abdc8e5f43ffa81f1ecaa5297a78 2024-11-11T20:41:55,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/92aeb96f93d0443692332a2d0c84e7fd is 50, key is test_row_0/B:col10/1731357715471/Put/seqid=0 2024-11-11T20:41:55,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742428_1604 (size=12301) 2024-11-11T20:41:56,100 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/2ba4cf0c5d914253a1af5ed1b5c47f2a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/2ba4cf0c5d914253a1af5ed1b5c47f2a 2024-11-11T20:41:56,104 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1be5d7c75f632327a07f28f72dd94fc1/C of 1be5d7c75f632327a07f28f72dd94fc1 into 2ba4cf0c5d914253a1af5ed1b5c47f2a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:41:56,104 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:56,104 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., storeName=1be5d7c75f632327a07f28f72dd94fc1/C, priority=12, startTime=1731357715262; duration=0sec 2024-11-11T20:41:56,104 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:41:56,104 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1be5d7c75f632327a07f28f72dd94fc1:C 2024-11-11T20:41:56,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/92aeb96f93d0443692332a2d0c84e7fd 2024-11-11T20:41:56,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/d1d72f8a4e454d39baef8764c15054b0 is 50, key is test_row_0/C:col10/1731357715471/Put/seqid=0 2024-11-11T20:41:56,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742429_1605 (size=12301) 2024-11-11T20:41:56,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/d1d72f8a4e454d39baef8764c15054b0 2024-11-11T20:41:56,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/00d5abdc8e5f43ffa81f1ecaa5297a78 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/00d5abdc8e5f43ffa81f1ecaa5297a78 2024-11-11T20:41:56,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/00d5abdc8e5f43ffa81f1ecaa5297a78, entries=150, sequenceid=456, filesize=12.0 K 2024-11-11T20:41:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/92aeb96f93d0443692332a2d0c84e7fd as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/92aeb96f93d0443692332a2d0c84e7fd 2024-11-11T20:41:56,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/92aeb96f93d0443692332a2d0c84e7fd, entries=150, sequenceid=456, filesize=12.0 K 2024-11-11T20:41:56,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/d1d72f8a4e454d39baef8764c15054b0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d1d72f8a4e454d39baef8764c15054b0 2024-11-11T20:41:56,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d1d72f8a4e454d39baef8764c15054b0, entries=150, sequenceid=456, filesize=12.0 K 2024-11-11T20:41:56,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=6.71 KB/6870 for 1be5d7c75f632327a07f28f72dd94fc1 in 1256ms, sequenceid=456, compaction requested=false 2024-11-11T20:41:56,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1be5d7c75f632327a07f28f72dd94fc1: 2024-11-11T20:41:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-11T20:41:56,761 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:02,932 INFO [Thread-2246 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2246. 2024-11-11T20:42:02,932 DEBUG [Thread-2246 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:02,932 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:02,932 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:02,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1544 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4629 rows 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1527 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4579 rows 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1545 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4634 rows 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1532 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4591 rows 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1539 2024-11-11T20:42:02,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4616 rows 2024-11-11T20:42:02,933 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:42:02,933 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:42:02,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:02,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:02,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:02,934 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:42:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-11T20:42:02,936 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357722936"}]},"ts":"1731357722936"} 2024-11-11T20:42:02,937 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:42:02,937 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:42:02,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:42:02,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1be5d7c75f632327a07f28f72dd94fc1, UNASSIGN}] 2024-11-11T20:42:02,939 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1be5d7c75f632327a07f28f72dd94fc1, UNASSIGN 2024-11-11T20:42:02,940 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=1be5d7c75f632327a07f28f72dd94fc1, regionState=CLOSING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:02,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1be5d7c75f632327a07f28f72dd94fc1, UNASSIGN because future has completed 2024-11-11T20:42:02,941 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:42:02,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:42:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-11T20:42:03,093 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:42:03,093 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:42:03,093 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing 1be5d7c75f632327a07f28f72dd94fc1, disabling compactions & flushes 2024-11-11T20:42:03,093 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:42:03,093 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:42:03,093 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. after waiting 0 ms 2024-11-11T20:42:03,093 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:42:03,093 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(2902): Flushing 1be5d7c75f632327a07f28f72dd94fc1 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-11T20:42:03,096 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/d50db4fca0fa44eeb13b2858ae50786e is 50, key is test_row_1/A:col10/1731357715490/Put/seqid=0 2024-11-11T20:42:03,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742430_1606 (size=9857) 2024-11-11T20:42:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-11T20:42:03,499 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/d50db4fca0fa44eeb13b2858ae50786e 2024-11-11T20:42:03,507 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/d144881b4d074547992550dffa4bf065 is 50, key is test_row_1/B:col10/1731357715490/Put/seqid=0 2024-11-11T20:42:03,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742431_1607 (size=9857) 2024-11-11T20:42:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-11T20:42:03,911 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/d144881b4d074547992550dffa4bf065 2024-11-11T20:42:03,916 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/cd9868f3a61a4cbcb3c3280f49a30589 is 50, key is test_row_1/C:col10/1731357715490/Put/seqid=0 2024-11-11T20:42:03,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742432_1608 (size=9857) 2024-11-11T20:42:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-11T20:42:04,320 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/cd9868f3a61a4cbcb3c3280f49a30589 2024-11-11T20:42:04,329 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/A/d50db4fca0fa44eeb13b2858ae50786e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/d50db4fca0fa44eeb13b2858ae50786e 2024-11-11T20:42:04,334 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/d50db4fca0fa44eeb13b2858ae50786e, entries=100, sequenceid=464, filesize=9.6 K 2024-11-11T20:42:04,335 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/B/d144881b4d074547992550dffa4bf065 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/d144881b4d074547992550dffa4bf065 2024-11-11T20:42:04,337 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/d144881b4d074547992550dffa4bf065, entries=100, sequenceid=464, filesize=9.6 K 2024-11-11T20:42:04,338 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/.tmp/C/cd9868f3a61a4cbcb3c3280f49a30589 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cd9868f3a61a4cbcb3c3280f49a30589 2024-11-11T20:42:04,340 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cd9868f3a61a4cbcb3c3280f49a30589, entries=100, sequenceid=464, filesize=9.6 K 2024-11-11T20:42:04,341 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 1be5d7c75f632327a07f28f72dd94fc1 in 1248ms, sequenceid=464, compaction requested=true 2024-11-11T20:42:04,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45037bc15d454ad8b57b4c92d509a30e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/bf3795304205436e92c6b5f649a9fa50, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c7ac20001020422ebaf3b69ea159eee7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c9675ae8b6e84a4b8606275a09a22ccb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ae0e5f65e45543d790eb4dc9a12acb78, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/6f63557d2ca247d1b55510df5fa14ea6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cbc95b4241574f768bd62a310d576bb7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/a96a0f5983134854b6d747e6b51753b4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ef9c4e96dbb4a47a4b92fcead8ce360, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/03264dfc529a44778b36aaf1b1262f2c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/557b0c1f4d004f5aa07ff42a53bebf98, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cdaff4cb82554d05bbfb7f457abe0225, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fe56f19882c44be39c69b8d6022cd643, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45f5ae9950804e3a95fd6e25b9b2f235, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/1a4d2d01de3d451bbc0c6b7d94f0634c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/f356bb38801143ecb136fd7fd934e274, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/27577745c8cf42c1ab828b45861efcac, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ebd753a8db0409f8f3575b88eab858d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/db26b200302e4e39951d3b5a04e62072, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/75bc1a6235d349e1b492e6adf70d29ad, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fc07b0720a8c4557861c4d9b764d785b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/465b984a9d65473bbc9076b6a7b8fefd, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/86f145087acc4f87a8c6f6a5324c0f54, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/96ecfb6bceab406bbd5225969ec52f0b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/302aa3b7dac7491e86597ccc976d1576, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/632c281770504b60b7d52af1e71f1e85] to archive 2024-11-11T20:42:04,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:42:04,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45037bc15d454ad8b57b4c92d509a30e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45037bc15d454ad8b57b4c92d509a30e 2024-11-11T20:42:04,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/bf3795304205436e92c6b5f649a9fa50 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/bf3795304205436e92c6b5f649a9fa50 2024-11-11T20:42:04,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c7ac20001020422ebaf3b69ea159eee7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c7ac20001020422ebaf3b69ea159eee7 2024-11-11T20:42:04,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c9675ae8b6e84a4b8606275a09a22ccb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/c9675ae8b6e84a4b8606275a09a22ccb 2024-11-11T20:42:04,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ae0e5f65e45543d790eb4dc9a12acb78 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/ae0e5f65e45543d790eb4dc9a12acb78 2024-11-11T20:42:04,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/6f63557d2ca247d1b55510df5fa14ea6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/6f63557d2ca247d1b55510df5fa14ea6 2024-11-11T20:42:04,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cbc95b4241574f768bd62a310d576bb7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cbc95b4241574f768bd62a310d576bb7 2024-11-11T20:42:04,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/a96a0f5983134854b6d747e6b51753b4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/a96a0f5983134854b6d747e6b51753b4 2024-11-11T20:42:04,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ef9c4e96dbb4a47a4b92fcead8ce360 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ef9c4e96dbb4a47a4b92fcead8ce360 2024-11-11T20:42:04,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/03264dfc529a44778b36aaf1b1262f2c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/03264dfc529a44778b36aaf1b1262f2c 2024-11-11T20:42:04,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/557b0c1f4d004f5aa07ff42a53bebf98 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/557b0c1f4d004f5aa07ff42a53bebf98 2024-11-11T20:42:04,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cdaff4cb82554d05bbfb7f457abe0225 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/cdaff4cb82554d05bbfb7f457abe0225 2024-11-11T20:42:04,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fe56f19882c44be39c69b8d6022cd643 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fe56f19882c44be39c69b8d6022cd643 2024-11-11T20:42:04,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45f5ae9950804e3a95fd6e25b9b2f235 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/45f5ae9950804e3a95fd6e25b9b2f235 2024-11-11T20:42:04,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/1a4d2d01de3d451bbc0c6b7d94f0634c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/1a4d2d01de3d451bbc0c6b7d94f0634c 2024-11-11T20:42:04,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/f356bb38801143ecb136fd7fd934e274 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/f356bb38801143ecb136fd7fd934e274 2024-11-11T20:42:04,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/27577745c8cf42c1ab828b45861efcac to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/27577745c8cf42c1ab828b45861efcac 2024-11-11T20:42:04,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ebd753a8db0409f8f3575b88eab858d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/7ebd753a8db0409f8f3575b88eab858d 2024-11-11T20:42:04,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/db26b200302e4e39951d3b5a04e62072 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/db26b200302e4e39951d3b5a04e62072 2024-11-11T20:42:04,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/75bc1a6235d349e1b492e6adf70d29ad to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/75bc1a6235d349e1b492e6adf70d29ad 2024-11-11T20:42:04,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fc07b0720a8c4557861c4d9b764d785b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/fc07b0720a8c4557861c4d9b764d785b 2024-11-11T20:42:04,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/465b984a9d65473bbc9076b6a7b8fefd to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/465b984a9d65473bbc9076b6a7b8fefd 2024-11-11T20:42:04,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/86f145087acc4f87a8c6f6a5324c0f54 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/86f145087acc4f87a8c6f6a5324c0f54 2024-11-11T20:42:04,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/96ecfb6bceab406bbd5225969ec52f0b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/96ecfb6bceab406bbd5225969ec52f0b 2024-11-11T20:42:04,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/302aa3b7dac7491e86597ccc976d1576 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/302aa3b7dac7491e86597ccc976d1576 2024-11-11T20:42:04,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/632c281770504b60b7d52af1e71f1e85 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/632c281770504b60b7d52af1e71f1e85 2024-11-11T20:42:04,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe504ffb14e64e64a76c0147b5251abc, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/26332e34c3404842a0a196c1539efe3c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/561f3c8185be4dafbee7946d3e819d1e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/843fdae3c65f4aadaa15c5e91dc72f3a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/16a22010030e44d19a6be1febbb0359f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e13d728a7f0486bb61aa2cc3aa6c0d8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7c06fd0484b54285af1a45957e4c7c0e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f46eba518cbf4373a3a153899d78f420, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/e3e557a07d0d481d95fa1066b23ed7eb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0629bfc86317426a9946d57395b007ff, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/b505008e6632416f9b579f3ac72863a1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7ab13ecf91de4c17b2d93b1db6328fed, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c40b0fc323e4fac93bedb190e237606, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/175b053c9a51485084ae54ff9f1aeb2e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/3dca793fd9334402b1688a1429f8bae4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f972b5fa15dc4fe49eb4f6e89f144a1d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/248aa9e7824a47399cb4b74f1dd630ef, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/44b159a7b3fc44a197d41b3c12aeb130, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0bf4fac2d3ee4499af01d8c94290668c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e61f0a502e04e9aa3a44637682e9c9d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/12c00e506179479e80839e0a9a34a5b2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe5ce28883e04fa8a67b84ca4ddd24d7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c4a78831bbf4c4e8c0f1d8960c442f4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/83b456d163134c54a93d5b76e00dc59e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/009fd171a0f14426a757539926911fdb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/136f8041f1594168b284567b0a289a15] to archive 2024-11-11T20:42:04,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:42:04,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe504ffb14e64e64a76c0147b5251abc to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe504ffb14e64e64a76c0147b5251abc 2024-11-11T20:42:04,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/26332e34c3404842a0a196c1539efe3c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/26332e34c3404842a0a196c1539efe3c 2024-11-11T20:42:04,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/561f3c8185be4dafbee7946d3e819d1e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/561f3c8185be4dafbee7946d3e819d1e 2024-11-11T20:42:04,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/843fdae3c65f4aadaa15c5e91dc72f3a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/843fdae3c65f4aadaa15c5e91dc72f3a 2024-11-11T20:42:04,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/16a22010030e44d19a6be1febbb0359f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/16a22010030e44d19a6be1febbb0359f 2024-11-11T20:42:04,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e13d728a7f0486bb61aa2cc3aa6c0d8 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e13d728a7f0486bb61aa2cc3aa6c0d8 2024-11-11T20:42:04,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7c06fd0484b54285af1a45957e4c7c0e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7c06fd0484b54285af1a45957e4c7c0e 2024-11-11T20:42:04,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f46eba518cbf4373a3a153899d78f420 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f46eba518cbf4373a3a153899d78f420 2024-11-11T20:42:04,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/e3e557a07d0d481d95fa1066b23ed7eb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/e3e557a07d0d481d95fa1066b23ed7eb 2024-11-11T20:42:04,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0629bfc86317426a9946d57395b007ff to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0629bfc86317426a9946d57395b007ff 2024-11-11T20:42:04,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/b505008e6632416f9b579f3ac72863a1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/b505008e6632416f9b579f3ac72863a1 2024-11-11T20:42:04,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7ab13ecf91de4c17b2d93b1db6328fed to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7ab13ecf91de4c17b2d93b1db6328fed 2024-11-11T20:42:04,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c40b0fc323e4fac93bedb190e237606 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c40b0fc323e4fac93bedb190e237606 2024-11-11T20:42:04,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/175b053c9a51485084ae54ff9f1aeb2e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/175b053c9a51485084ae54ff9f1aeb2e 2024-11-11T20:42:04,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/3dca793fd9334402b1688a1429f8bae4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/3dca793fd9334402b1688a1429f8bae4 2024-11-11T20:42:04,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f972b5fa15dc4fe49eb4f6e89f144a1d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/f972b5fa15dc4fe49eb4f6e89f144a1d 2024-11-11T20:42:04,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/248aa9e7824a47399cb4b74f1dd630ef to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/248aa9e7824a47399cb4b74f1dd630ef 2024-11-11T20:42:04,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/44b159a7b3fc44a197d41b3c12aeb130 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/44b159a7b3fc44a197d41b3c12aeb130 2024-11-11T20:42:04,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0bf4fac2d3ee4499af01d8c94290668c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/0bf4fac2d3ee4499af01d8c94290668c 2024-11-11T20:42:04,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e61f0a502e04e9aa3a44637682e9c9d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/7e61f0a502e04e9aa3a44637682e9c9d 2024-11-11T20:42:04,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/12c00e506179479e80839e0a9a34a5b2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/12c00e506179479e80839e0a9a34a5b2 2024-11-11T20:42:04,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe5ce28883e04fa8a67b84ca4ddd24d7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/fe5ce28883e04fa8a67b84ca4ddd24d7 2024-11-11T20:42:04,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c4a78831bbf4c4e8c0f1d8960c442f4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/1c4a78831bbf4c4e8c0f1d8960c442f4 2024-11-11T20:42:04,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/83b456d163134c54a93d5b76e00dc59e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/83b456d163134c54a93d5b76e00dc59e 2024-11-11T20:42:04,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/009fd171a0f14426a757539926911fdb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/009fd171a0f14426a757539926911fdb 2024-11-11T20:42:04,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/136f8041f1594168b284567b0a289a15 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/136f8041f1594168b284567b0a289a15 2024-11-11T20:42:04,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1862030a86fd4873824ba76242e7bed1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/3c8a502fa5954d1ba9f861da7971b7ae, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/800e40b491c5418b86e4e2e35734490a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/19039b1e84ef455885f51b37b1a3170d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7e076e71c9874fa6ac427d425d32435a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/a6f245a7b61d4e0b813584f6d73ecddb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/e6fc609b5b5b41bda386425cc717a9f4, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c90a512b0d0b43a3a99622aedfc61391, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/dc9a1d0784974bfeb95ed55dc912259a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/920edd3d323b4eb3b4796cd449523a0b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/bbbea0213cf440268ba57ddad3c90854, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7fe3a8e3b28743b39d3983d9f184b516, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/355bcdea86e040098fa8b65b04737476, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d863d59cb1244a9b80a9d4d3bd853029, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/258bbf78c89f4e6ab04b4777b4db21bb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/9f0276987b944ae29ff6f03bfd0dfda2, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/aa610fe7d41b40dcb3b325379b9abf14, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/40c2145913374c8abc6367b1ca3f4942, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/91af99860e93464abc5248b5085fe9df, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c8aa6ccc848d48dea0f9abcb568e4f3a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b25a26ec5a4c47cfa94dc49d008d4c5a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c05b3b70cb8f47aa98401b4d274d209c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4e1e67f959a346499a05e20933aac699, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1b65443e2b6d46c5af2b5f0b04c32492, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/fa6ec5df972543ef99a2d76e170e039c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cb1989cfaf644d7a90f9a89c6ad77743, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/95b0b27769d04c00ac18991778035d89, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/003dcadd078f40f28d3445f77c4a04c5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/ec2673d47f34421fa7505f33b858b45d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b327c006a7e844189fc734dee1544d92, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4c038ef69af84f7aae18ee5eaa60cdac] to archive 2024-11-11T20:42:04,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:42:04,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1862030a86fd4873824ba76242e7bed1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1862030a86fd4873824ba76242e7bed1 2024-11-11T20:42:04,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/3c8a502fa5954d1ba9f861da7971b7ae to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/3c8a502fa5954d1ba9f861da7971b7ae 2024-11-11T20:42:04,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/800e40b491c5418b86e4e2e35734490a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/800e40b491c5418b86e4e2e35734490a 2024-11-11T20:42:04,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/19039b1e84ef455885f51b37b1a3170d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/19039b1e84ef455885f51b37b1a3170d 2024-11-11T20:42:04,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7e076e71c9874fa6ac427d425d32435a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7e076e71c9874fa6ac427d425d32435a 2024-11-11T20:42:04,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/a6f245a7b61d4e0b813584f6d73ecddb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/a6f245a7b61d4e0b813584f6d73ecddb 2024-11-11T20:42:04,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/e6fc609b5b5b41bda386425cc717a9f4 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/e6fc609b5b5b41bda386425cc717a9f4 2024-11-11T20:42:04,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c90a512b0d0b43a3a99622aedfc61391 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c90a512b0d0b43a3a99622aedfc61391 2024-11-11T20:42:04,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/dc9a1d0784974bfeb95ed55dc912259a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/dc9a1d0784974bfeb95ed55dc912259a 2024-11-11T20:42:04,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/920edd3d323b4eb3b4796cd449523a0b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/920edd3d323b4eb3b4796cd449523a0b 2024-11-11T20:42:04,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/bbbea0213cf440268ba57ddad3c90854 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/bbbea0213cf440268ba57ddad3c90854 2024-11-11T20:42:04,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7fe3a8e3b28743b39d3983d9f184b516 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/7fe3a8e3b28743b39d3983d9f184b516 2024-11-11T20:42:04,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/355bcdea86e040098fa8b65b04737476 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/355bcdea86e040098fa8b65b04737476 2024-11-11T20:42:04,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d863d59cb1244a9b80a9d4d3bd853029 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d863d59cb1244a9b80a9d4d3bd853029 2024-11-11T20:42:04,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/258bbf78c89f4e6ab04b4777b4db21bb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/258bbf78c89f4e6ab04b4777b4db21bb 2024-11-11T20:42:04,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/9f0276987b944ae29ff6f03bfd0dfda2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/9f0276987b944ae29ff6f03bfd0dfda2 2024-11-11T20:42:04,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/aa610fe7d41b40dcb3b325379b9abf14 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/aa610fe7d41b40dcb3b325379b9abf14 2024-11-11T20:42:04,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/40c2145913374c8abc6367b1ca3f4942 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/40c2145913374c8abc6367b1ca3f4942 2024-11-11T20:42:04,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/91af99860e93464abc5248b5085fe9df to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/91af99860e93464abc5248b5085fe9df 2024-11-11T20:42:04,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c8aa6ccc848d48dea0f9abcb568e4f3a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c8aa6ccc848d48dea0f9abcb568e4f3a 2024-11-11T20:42:04,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b25a26ec5a4c47cfa94dc49d008d4c5a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b25a26ec5a4c47cfa94dc49d008d4c5a 2024-11-11T20:42:04,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c05b3b70cb8f47aa98401b4d274d209c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/c05b3b70cb8f47aa98401b4d274d209c 2024-11-11T20:42:04,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4e1e67f959a346499a05e20933aac699 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4e1e67f959a346499a05e20933aac699 2024-11-11T20:42:04,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1b65443e2b6d46c5af2b5f0b04c32492 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/1b65443e2b6d46c5af2b5f0b04c32492 2024-11-11T20:42:04,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/fa6ec5df972543ef99a2d76e170e039c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/fa6ec5df972543ef99a2d76e170e039c 2024-11-11T20:42:04,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cb1989cfaf644d7a90f9a89c6ad77743 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cb1989cfaf644d7a90f9a89c6ad77743 2024-11-11T20:42:04,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/95b0b27769d04c00ac18991778035d89 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/95b0b27769d04c00ac18991778035d89 2024-11-11T20:42:04,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/003dcadd078f40f28d3445f77c4a04c5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/003dcadd078f40f28d3445f77c4a04c5 2024-11-11T20:42:04,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/ec2673d47f34421fa7505f33b858b45d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/ec2673d47f34421fa7505f33b858b45d 2024-11-11T20:42:04,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b327c006a7e844189fc734dee1544d92 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/b327c006a7e844189fc734dee1544d92 2024-11-11T20:42:04,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4c038ef69af84f7aae18ee5eaa60cdac to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/4c038ef69af84f7aae18ee5eaa60cdac 2024-11-11T20:42:04,406 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/recovered.edits/467.seqid, newMaxSeqId=467, maxSeqId=1 2024-11-11T20:42:04,407 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. 2024-11-11T20:42:04,407 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for 1be5d7c75f632327a07f28f72dd94fc1: Waiting for close lock at 1731357723093Running coprocessor pre-close hooks at 1731357723093Disabling compacts and flushes for region at 1731357723093Disabling writes for close at 1731357723093Obtaining lock to block concurrent updates at 1731357723093Preparing flush snapshotting stores in 1be5d7c75f632327a07f28f72dd94fc1 at 1731357723093Finished memstore snapshotting TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731357723093Flushing stores of TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1. at 1731357723094 (+1 ms)Flushing 1be5d7c75f632327a07f28f72dd94fc1/A: creating writer at 1731357723094Flushing 1be5d7c75f632327a07f28f72dd94fc1/A: appending metadata at 1731357723095 (+1 ms)Flushing 1be5d7c75f632327a07f28f72dd94fc1/A: closing flushed file at 1731357723095Flushing 1be5d7c75f632327a07f28f72dd94fc1/B: creating writer at 1731357723503 (+408 ms)Flushing 1be5d7c75f632327a07f28f72dd94fc1/B: appending metadata at 1731357723506 (+3 ms)Flushing 1be5d7c75f632327a07f28f72dd94fc1/B: closing flushed file at 1731357723506Flushing 1be5d7c75f632327a07f28f72dd94fc1/C: creating writer at 1731357723914 (+408 ms)Flushing 1be5d7c75f632327a07f28f72dd94fc1/C: appending metadata at 1731357723916 (+2 ms)Flushing 1be5d7c75f632327a07f28f72dd94fc1/C: closing flushed file at 1731357723916Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e21882b: reopening flushed file at 1731357724327 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ce88154: reopening flushed file at 1731357724334 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bbb3ecb: reopening flushed file at 1731357724337 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 1be5d7c75f632327a07f28f72dd94fc1 in 1248ms, sequenceid=464, compaction requested=true at 1731357724341 (+4 ms)Writing region close event to WAL at 1731357724404 (+63 ms)Running coprocessor post-close hooks at 1731357724407 (+3 ms)Closed at 1731357724407 2024-11-11T20:42:04,409 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed 1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:42:04,409 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=1be5d7c75f632327a07f28f72dd94fc1, regionState=CLOSED 2024-11-11T20:42:04,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:42:04,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-11T20:42:04,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure 1be5d7c75f632327a07f28f72dd94fc1, server=51ca66f7ee3c,44271,1731357576703 in 1.4690 sec 2024-11-11T20:42:04,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T20:42:04,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1be5d7c75f632327a07f28f72dd94fc1, UNASSIGN in 1.4730 sec 2024-11-11T20:42:04,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-11T20:42:04,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4750 sec 2024-11-11T20:42:04,415 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357724415"}]},"ts":"1731357724415"} 2024-11-11T20:42:04,416 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:42:04,416 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:42:04,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4830 sec 2024-11-11T20:42:05,053 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:42:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-11T20:42:05,091 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:05,092 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:42:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=157, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:05,094 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=157, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=157 2024-11-11T20:42:05,095 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=157, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:05,098 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:42:05,100 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/recovered.edits] 2024-11-11T20:42:05,104 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/00d5abdc8e5f43ffa81f1ecaa5297a78 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/00d5abdc8e5f43ffa81f1ecaa5297a78 2024-11-11T20:42:05,106 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/a9fbeb0e6e9245a29ad9bae0473c14a8 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/a9fbeb0e6e9245a29ad9bae0473c14a8 2024-11-11T20:42:05,107 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/d50db4fca0fa44eeb13b2858ae50786e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/A/d50db4fca0fa44eeb13b2858ae50786e 2024-11-11T20:42:05,110 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/2ca20b28712040dab51a103fffdd4ed6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/2ca20b28712040dab51a103fffdd4ed6 2024-11-11T20:42:05,111 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/92aeb96f93d0443692332a2d0c84e7fd to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/92aeb96f93d0443692332a2d0c84e7fd 2024-11-11T20:42:05,113 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/d144881b4d074547992550dffa4bf065 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/B/d144881b4d074547992550dffa4bf065 2024-11-11T20:42:05,116 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/2ba4cf0c5d914253a1af5ed1b5c47f2a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/2ba4cf0c5d914253a1af5ed1b5c47f2a 2024-11-11T20:42:05,117 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cd9868f3a61a4cbcb3c3280f49a30589 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/cd9868f3a61a4cbcb3c3280f49a30589 2024-11-11T20:42:05,119 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d1d72f8a4e454d39baef8764c15054b0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/C/d1d72f8a4e454d39baef8764c15054b0 2024-11-11T20:42:05,122 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/recovered.edits/467.seqid to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1/recovered.edits/467.seqid 2024-11-11T20:42:05,123 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/1be5d7c75f632327a07f28f72dd94fc1 2024-11-11T20:42:05,123 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:42:05,125 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=157, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:05,128 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:42:05,131 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:42:05,133 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=157, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:05,133 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:42:05,133 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731357725133"}]},"ts":"9223372036854775807"} 2024-11-11T20:42:05,136 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:42:05,136 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1be5d7c75f632327a07f28f72dd94fc1, NAME => 'TestAcidGuarantees,,1731357692935.1be5d7c75f632327a07f28f72dd94fc1.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:42:05,136 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:42:05,136 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731357725136"}]},"ts":"9223372036854775807"} 2024-11-11T20:42:05,138 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:42:05,139 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=157, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:05,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-11T20:42:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=157 2024-11-11T20:42:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:42:05,204 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:05,213 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 242), OpenFileDescriptor=445 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=435 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=4367 (was 4663) 2024-11-11T20:42:05,222 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=435, ProcessCount=11, AvailableMemoryMB=4367 2024-11-11T20:42:05,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:42:05,224 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:42:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:05,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:42:05,226 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:05,226 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 158 2024-11-11T20:42:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T20:42:05,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:42:05,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742433_1609 (size=959) 2024-11-11T20:42:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T20:42:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T20:42:05,633 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30 2024-11-11T20:42:05,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742434_1610 (size=53) 2024-11-11T20:42:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T20:42:06,037 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:42:06,037 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fe8aa7c45afd5cfd9441128b14c2ef12, disabling compactions & flushes 2024-11-11T20:42:06,037 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:06,037 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:06,037 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. after waiting 0 ms 2024-11-11T20:42:06,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:06,038 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:06,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe8aa7c45afd5cfd9441128b14c2ef12: Waiting for close lock at 1731357726037Disabling compacts and flushes for region at 1731357726037Disabling writes for close at 1731357726037Writing region close event to WAL at 1731357726038 (+1 ms)Closed at 1731357726038 2024-11-11T20:42:06,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:42:06,039 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731357726039"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731357726039"}]},"ts":"1731357726039"} 2024-11-11T20:42:06,040 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:42:06,040 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:42:06,041 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357726041"}]},"ts":"1731357726041"} 2024-11-11T20:42:06,042 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:42:06,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, ASSIGN}] 2024-11-11T20:42:06,043 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, ASSIGN 2024-11-11T20:42:06,044 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44271,1731357576703; forceNewPlan=false, retain=false 2024-11-11T20:42:06,195 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=fe8aa7c45afd5cfd9441128b14c2ef12, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:06,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, ASSIGN because future has completed 2024-11-11T20:42:06,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:42:06,351 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:06,351 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(7752): Opening region: {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:42:06,352 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,352 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:42:06,352 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(7794): checking encryption for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,352 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(7797): checking classloading for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,353 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,354 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe8aa7c45afd5cfd9441128b14c2ef12 columnFamilyName A 2024-11-11T20:42:06,354 DEBUG [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:06,355 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(327): Store=fe8aa7c45afd5cfd9441128b14c2ef12/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:42:06,355 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,356 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe8aa7c45afd5cfd9441128b14c2ef12 columnFamilyName B 2024-11-11T20:42:06,356 DEBUG [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:06,356 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(327): Store=fe8aa7c45afd5cfd9441128b14c2ef12/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:42:06,356 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,357 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe8aa7c45afd5cfd9441128b14c2ef12 columnFamilyName C 2024-11-11T20:42:06,357 DEBUG [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:06,358 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(327): Store=fe8aa7c45afd5cfd9441128b14c2ef12/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:42:06,358 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1038): replaying wal for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,358 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,358 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,359 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1048): stopping wal replay for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,359 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1060): Cleaning up temporary data for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,359 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:42:06,361 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1093): writing seq id for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T20:42:06,362 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:42:06,363 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1114): Opened fe8aa7c45afd5cfd9441128b14c2ef12; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74353146, jitterRate=0.10794821381568909}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:42:06,363 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:06,363 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1006): Region open journal for fe8aa7c45afd5cfd9441128b14c2ef12: Running coprocessor pre-open hook at 1731357726352Writing region info on filesystem at 1731357726352Initializing all the Stores at 1731357726353 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357726353Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357726353Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357726353Cleaning up temporary data from old regions at 1731357726359 (+6 ms)Running coprocessor post-open hooks at 1731357726363 (+4 ms)Region opened successfully at 1731357726363 2024-11-11T20:42:06,364 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., pid=160, masterSystemTime=1731357726348 2024-11-11T20:42:06,365 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:06,366 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:06,366 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=fe8aa7c45afd5cfd9441128b14c2ef12, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:06,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:42:06,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-11T20:42:06,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; OpenRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 in 172 msec 2024-11-11T20:42:06,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T20:42:06,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, ASSIGN in 327 msec 2024-11-11T20:42:06,371 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:42:06,371 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357726371"}]},"ts":"1731357726371"} 2024-11-11T20:42:06,372 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:42:06,373 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:42:06,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-11T20:42:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T20:42:07,372 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:07,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:07,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:07,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:07,375 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1b93a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:07,378 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265f3a33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:07,380 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:07,382 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:07,383 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:42:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,35885,1731357576039 2024-11-11T20:42:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b8f075b 2024-11-11T20:42:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:42:07,385 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:42:07,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:42:07,386 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:42:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=161, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:07,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742435_1611 (size=995) 2024-11-11T20:42:07,800 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T20:42:07,800 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T20:42:07,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:42:07,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, REOPEN/MOVE}] 2024-11-11T20:42:07,806 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, REOPEN/MOVE 2024-11-11T20:42:07,807 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=fe8aa7c45afd5cfd9441128b14c2ef12, regionState=CLOSING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:07,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, REOPEN/MOVE because future has completed 2024-11-11T20:42:07,808 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:42:07,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:42:07,960 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(122): Close fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:07,960 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:42:07,960 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1722): Closing fe8aa7c45afd5cfd9441128b14c2ef12, disabling compactions & flushes 2024-11-11T20:42:07,960 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:07,960 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:07,960 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. after waiting 0 ms 2024-11-11T20:42:07,960 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:07,963 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:42:07,963 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:07,963 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1676): Region close journal for fe8aa7c45afd5cfd9441128b14c2ef12: Waiting for close lock at 1731357727960Running coprocessor pre-close hooks at 1731357727960Disabling compacts and flushes for region at 1731357727960Disabling writes for close at 1731357727960Writing region close event to WAL at 1731357727960Running coprocessor post-close hooks at 1731357727963 (+3 ms)Closed at 1731357727963 2024-11-11T20:42:07,963 WARN [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegionServer(3299): Not adding moved region record: fe8aa7c45afd5cfd9441128b14c2ef12 to self. 2024-11-11T20:42:07,965 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(157): Closed fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:07,965 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=fe8aa7c45afd5cfd9441128b14c2ef12, regionState=CLOSED 2024-11-11T20:42:07,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:42:07,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-11T20:42:07,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; CloseRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 in 159 msec 2024-11-11T20:42:07,969 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, REOPEN/MOVE; state=CLOSED, location=51ca66f7ee3c,44271,1731357576703; forceNewPlan=false, retain=true 2024-11-11T20:42:08,120 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=fe8aa7c45afd5cfd9441128b14c2ef12, regionState=OPENING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, REOPEN/MOVE because future has completed 2024-11-11T20:42:08,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=163, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:42:08,276 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,276 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(7752): Opening region: {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:42:08,276 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,276 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:42:08,276 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(7794): checking encryption for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,276 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(7797): checking classloading for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,278 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,278 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe8aa7c45afd5cfd9441128b14c2ef12 columnFamilyName A 2024-11-11T20:42:08,280 DEBUG [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:08,280 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(327): Store=fe8aa7c45afd5cfd9441128b14c2ef12/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:42:08,281 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,282 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe8aa7c45afd5cfd9441128b14c2ef12 columnFamilyName B 2024-11-11T20:42:08,282 DEBUG [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:08,282 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(327): Store=fe8aa7c45afd5cfd9441128b14c2ef12/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:42:08,283 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,283 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe8aa7c45afd5cfd9441128b14c2ef12 columnFamilyName C 2024-11-11T20:42:08,283 DEBUG [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:08,284 INFO [StoreOpener-fe8aa7c45afd5cfd9441128b14c2ef12-1 {}] regionserver.HStore(327): Store=fe8aa7c45afd5cfd9441128b14c2ef12/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:42:08,284 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1038): replaying wal for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,285 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,286 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,286 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1048): stopping wal replay for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,286 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1060): Cleaning up temporary data for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,287 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:42:08,288 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1093): writing seq id for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,288 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1114): Opened fe8aa7c45afd5cfd9441128b14c2ef12; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63750944, jitterRate=-0.05003690719604492}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:42:08,288 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,289 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1006): Region open journal for fe8aa7c45afd5cfd9441128b14c2ef12: Running coprocessor pre-open hook at 1731357728276Writing region info on filesystem at 1731357728276Initializing all the Stores at 1731357728277 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357728277Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357728277Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731357728277Cleaning up temporary data from old regions at 1731357728286 (+9 ms)Running coprocessor post-open hooks at 1731357728288 (+2 ms)Region opened successfully at 1731357728289 (+1 ms) 2024-11-11T20:42:08,289 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., pid=165, masterSystemTime=1731357728273 2024-11-11T20:42:08,291 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,291 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,291 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=fe8aa7c45afd5cfd9441128b14c2ef12, regionState=OPEN, openSeqNum=5, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=163, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:42:08,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=163 2024-11-11T20:42:08,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=163, state=SUCCESS, hasLock=false; OpenRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 in 172 msec 2024-11-11T20:42:08,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-11T20:42:08,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, REOPEN/MOVE in 489 msec 2024-11-11T20:42:08,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-11T20:42:08,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-11T20:42:08,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 911 msec 2024-11-11T20:42:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161 2024-11-11T20:42:08,300 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:08,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,301 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa19c87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,303 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c71c535, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,304 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d37597f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,305 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177aed16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,307 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419c9b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,308 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39adff4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,309 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5583b456, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,310 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3c0e99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,312 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25676347, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,313 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d65ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,315 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a025d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,316 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd0f0d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,317 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f470959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,319 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150195b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,320 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a92c6c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,321 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b900e2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,323 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25e8c26a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a3ef50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,35885,-1 for getting cluster id 2024-11-11T20:42:08,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:42:08,325 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7eb98061-214b-40f9-8a48-d23c5a1d1708' 2024-11-11T20:42:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:42:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7eb98061-214b-40f9-8a48-d23c5a1d1708" 2024-11-11T20:42:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@232685b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,35885,-1] 2024-11-11T20:42:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:42:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:08,326 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:42:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed6f4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:42:08,327 DEBUG [Thread-2699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,327 DEBUG [Thread-2705 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,327 DEBUG [Thread-2703 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,327 DEBUG [Thread-2696 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,328 DEBUG [Thread-2688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,328 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:42:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,328 DEBUG [Thread-2690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,328 DEBUG [Thread-2707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,330 DEBUG [Thread-2692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,330 DEBUG [Thread-2701 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,330 DEBUG [Thread-2690 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,331 DEBUG [Thread-2690 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-11T20:42:08,331 DEBUG [Thread-2694 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:42:08,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T20:42:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44271,1731357576703, seqNum=-1] 2024-11-11T20:42:08,332 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:42:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:42:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,333 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:42:08,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:42:08,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,334 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:42:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5] 2024-11-11T20:42:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,371 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:42:08,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T20:42:08,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110b8ab87f3d4445079ed19f069c9d6830_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357728338/Put/seqid=0 2024-11-11T20:42:08,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357788394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357788395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357788397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357788397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357788397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742436_1612 (size=12154) 2024-11-11T20:42:08,414 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:08,419 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110b8ab87f3d4445079ed19f069c9d6830_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110b8ab87f3d4445079ed19f069c9d6830_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:08,420 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/4bcc64c998834fb38e86dc887fa32fd9, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:08,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/4bcc64c998834fb38e86dc887fa32fd9 is 175, key is test_row_0/A:col10/1731357728338/Put/seqid=0 2024-11-11T20:42:08,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742437_1613 (size=30955) 2024-11-11T20:42:08,427 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/4bcc64c998834fb38e86dc887fa32fd9 2024-11-11T20:42:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T20:42:08,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/56fe20d043604f0ab958ff52bec5e7ac is 50, key is test_row_0/B:col10/1731357728338/Put/seqid=0 2024-11-11T20:42:08,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742438_1614 (size=12001) 2024-11-11T20:42:08,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/56fe20d043604f0ab958ff52bec5e7ac 2024-11-11T20:42:08,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/8e6e9802ed3044a399d1ca76528d9e01 is 50, key is test_row_0/C:col10/1731357728338/Put/seqid=0 2024-11-11T20:42:08,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742439_1615 (size=12001) 2024-11-11T20:42:08,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T20:42:08,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:08,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:08,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:08,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357788511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357788511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357788511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357788512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357788512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T20:42:08,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:08,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:08,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T20:42:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357788720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357788720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357788720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357788720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357788720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:08,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T20:42:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:08,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/8e6e9802ed3044a399d1ca76528d9e01 2024-11-11T20:42:08,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/4bcc64c998834fb38e86dc887fa32fd9 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4bcc64c998834fb38e86dc887fa32fd9 2024-11-11T20:42:08,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4bcc64c998834fb38e86dc887fa32fd9, entries=150, sequenceid=16, filesize=30.2 K 2024-11-11T20:42:08,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/56fe20d043604f0ab958ff52bec5e7ac as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/56fe20d043604f0ab958ff52bec5e7ac 2024-11-11T20:42:08,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/56fe20d043604f0ab958ff52bec5e7ac, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T20:42:08,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/8e6e9802ed3044a399d1ca76528d9e01 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8e6e9802ed3044a399d1ca76528d9e01 2024-11-11T20:42:08,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8e6e9802ed3044a399d1ca76528d9e01, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T20:42:08,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for fe8aa7c45afd5cfd9441128b14c2ef12 in 524ms, sequenceid=16, compaction requested=false 2024-11-11T20:42:08,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:08,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T20:42:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:08,944 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T20:42:08,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111da005ae1e674414895ced8f288fb2a98_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357728395/Put/seqid=0 2024-11-11T20:42:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T20:42:08,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742440_1616 (size=12154) 2024-11-11T20:42:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:09,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:09,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357789037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357789037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357789038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357789039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357789039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357789150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357789150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357789151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357789151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357789151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357789361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357789361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357789361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357789361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357789361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:09,367 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111da005ae1e674414895ced8f288fb2a98_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111da005ae1e674414895ced8f288fb2a98_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:09,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/ad660933799546b993d0cc484d11ded8, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:09,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/ad660933799546b993d0cc484d11ded8 is 175, key is test_row_0/A:col10/1731357728395/Put/seqid=0 2024-11-11T20:42:09,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742441_1617 (size=30955) 2024-11-11T20:42:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T20:42:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357789670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357789670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357789670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357789670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357789670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:09,775 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/ad660933799546b993d0cc484d11ded8 2024-11-11T20:42:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/9f7b3186800d4dc0be5430a9735baf4b is 50, key is test_row_0/B:col10/1731357728395/Put/seqid=0 2024-11-11T20:42:09,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742442_1618 (size=12001) 2024-11-11T20:42:09,926 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:42:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357790180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357790180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357790180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357790181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357790181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:10,183 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/9f7b3186800d4dc0be5430a9735baf4b 2024-11-11T20:42:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/f298160083074c7c8c12fc95ceb5d7fa is 50, key is test_row_0/C:col10/1731357728395/Put/seqid=0 2024-11-11T20:42:10,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742443_1619 (size=12001) 2024-11-11T20:42:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T20:42:10,593 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/f298160083074c7c8c12fc95ceb5d7fa 2024-11-11T20:42:10,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/ad660933799546b993d0cc484d11ded8 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ad660933799546b993d0cc484d11ded8 2024-11-11T20:42:10,599 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ad660933799546b993d0cc484d11ded8, entries=150, sequenceid=42, filesize=30.2 K 2024-11-11T20:42:10,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/9f7b3186800d4dc0be5430a9735baf4b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f7b3186800d4dc0be5430a9735baf4b 2024-11-11T20:42:10,602 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f7b3186800d4dc0be5430a9735baf4b, entries=150, sequenceid=42, filesize=11.7 K 2024-11-11T20:42:10,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/f298160083074c7c8c12fc95ceb5d7fa as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f298160083074c7c8c12fc95ceb5d7fa 2024-11-11T20:42:10,605 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f298160083074c7c8c12fc95ceb5d7fa, entries=150, sequenceid=42, filesize=11.7 K 2024-11-11T20:42:10,606 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1662ms, sequenceid=42, compaction requested=false 2024-11-11T20:42:10,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:10,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:10,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-11T20:42:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-11T20:42:10,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-11T20:42:10,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2740 sec 2024-11-11T20:42:10,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.2820 sec 2024-11-11T20:42:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:11,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:42:11,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:42:11,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ecc71c6613f04b728dfed0e1bd2638c8_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357731192/Put/seqid=0 2024-11-11T20:42:11,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742444_1620 (size=17034) 2024-11-11T20:42:11,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357791245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357791245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357791246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357791246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357791247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357791360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357791360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357791360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357791360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357791360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357791570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357791570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357791570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357791570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357791570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,604 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:11,606 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ecc71c6613f04b728dfed0e1bd2638c8_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ecc71c6613f04b728dfed0e1bd2638c8_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:11,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/907d3a5eafcd4908a598186aa31522a1, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:11,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/907d3a5eafcd4908a598186aa31522a1 is 175, key is test_row_0/A:col10/1731357731192/Put/seqid=0 2024-11-11T20:42:11,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742445_1621 (size=48139) 2024-11-11T20:42:11,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357791881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357791881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357791881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357791881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357791881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:12,010 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/907d3a5eafcd4908a598186aa31522a1 2024-11-11T20:42:12,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/c4cc5e0723094b138e177ceaaf7282a0 is 50, key is test_row_0/B:col10/1731357731192/Put/seqid=0 2024-11-11T20:42:12,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742446_1622 (size=12001) 2024-11-11T20:42:12,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357792391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357792391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357792391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357792391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357792391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:12,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/c4cc5e0723094b138e177ceaaf7282a0 2024-11-11T20:42:12,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/206739dcfe2e4cfc96e5e8d962d0575f is 50, key is test_row_0/C:col10/1731357731192/Put/seqid=0 2024-11-11T20:42:12,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742447_1623 (size=12001) 2024-11-11T20:42:12,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/206739dcfe2e4cfc96e5e8d962d0575f 2024-11-11T20:42:12,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/907d3a5eafcd4908a598186aa31522a1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/907d3a5eafcd4908a598186aa31522a1 2024-11-11T20:42:12,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/907d3a5eafcd4908a598186aa31522a1, entries=250, sequenceid=56, filesize=47.0 K 2024-11-11T20:42:12,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/c4cc5e0723094b138e177ceaaf7282a0 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c4cc5e0723094b138e177ceaaf7282a0 2024-11-11T20:42:12,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c4cc5e0723094b138e177ceaaf7282a0, entries=150, sequenceid=56, filesize=11.7 K 2024-11-11T20:42:12,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/206739dcfe2e4cfc96e5e8d962d0575f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/206739dcfe2e4cfc96e5e8d962d0575f 2024-11-11T20:42:12,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/206739dcfe2e4cfc96e5e8d962d0575f, entries=150, sequenceid=56, filesize=11.7 K 2024-11-11T20:42:12,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1282ms, sequenceid=56, compaction requested=true 2024-11-11T20:42:12,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:12,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:42:12,474 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:12,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:12,474 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:12,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:42:12,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:12,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:42:12,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:12,475 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:12,475 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/B is initiating minor compaction (all files) 2024-11-11T20:42:12,475 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/B in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:12,475 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/56fe20d043604f0ab958ff52bec5e7ac, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f7b3186800d4dc0be5430a9735baf4b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c4cc5e0723094b138e177ceaaf7282a0] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=35.2 K 2024-11-11T20:42:12,475 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:12,475 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 56fe20d043604f0ab958ff52bec5e7ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731357728338 2024-11-11T20:42:12,475 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/A is initiating minor compaction (all files) 2024-11-11T20:42:12,475 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/A in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:12,475 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4bcc64c998834fb38e86dc887fa32fd9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ad660933799546b993d0cc484d11ded8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/907d3a5eafcd4908a598186aa31522a1] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=107.5 K 2024-11-11T20:42:12,476 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:12,476 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4bcc64c998834fb38e86dc887fa32fd9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ad660933799546b993d0cc484d11ded8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/907d3a5eafcd4908a598186aa31522a1] 2024-11-11T20:42:12,476 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7b3186800d4dc0be5430a9735baf4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731357728394 2024-11-11T20:42:12,476 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bcc64c998834fb38e86dc887fa32fd9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731357728338 2024-11-11T20:42:12,476 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c4cc5e0723094b138e177ceaaf7282a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731357729038 2024-11-11T20:42:12,476 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad660933799546b993d0cc484d11ded8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731357728394 2024-11-11T20:42:12,476 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 907d3a5eafcd4908a598186aa31522a1, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731357729038 2024-11-11T20:42:12,480 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:12,482 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#B#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:12,483 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/17e42c93c5474fc1a33ac19d30b22639 is 50, key is test_row_0/B:col10/1731357731192/Put/seqid=0 2024-11-11T20:42:12,488 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110fbc482fd4ab479dba5c48560589b99e_fe8aa7c45afd5cfd9441128b14c2ef12 store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:12,490 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110fbc482fd4ab479dba5c48560589b99e_fe8aa7c45afd5cfd9441128b14c2ef12, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:12,490 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110fbc482fd4ab479dba5c48560589b99e_fe8aa7c45afd5cfd9441128b14c2ef12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:12,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742448_1624 (size=12104) 2024-11-11T20:42:12,494 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/17e42c93c5474fc1a33ac19d30b22639 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/17e42c93c5474fc1a33ac19d30b22639 2024-11-11T20:42:12,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742449_1625 (size=4469) 2024-11-11T20:42:12,498 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#A#compaction#531 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:12,498 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/e7865388b39644b898f796f23b7b64ad is 175, key is test_row_0/A:col10/1731357731192/Put/seqid=0 2024-11-11T20:42:12,499 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/B of fe8aa7c45afd5cfd9441128b14c2ef12 into 17e42c93c5474fc1a33ac19d30b22639(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:12,499 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:12,499 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/B, priority=13, startTime=1731357732474; duration=0sec 2024-11-11T20:42:12,499 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:12,499 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:B 2024-11-11T20:42:12,499 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:12,500 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:12,500 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/C is initiating minor compaction (all files) 2024-11-11T20:42:12,500 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/C in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:12,500 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8e6e9802ed3044a399d1ca76528d9e01, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f298160083074c7c8c12fc95ceb5d7fa, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/206739dcfe2e4cfc96e5e8d962d0575f] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=35.2 K 2024-11-11T20:42:12,500 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6e9802ed3044a399d1ca76528d9e01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731357728338 2024-11-11T20:42:12,500 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f298160083074c7c8c12fc95ceb5d7fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731357728394 2024-11-11T20:42:12,500 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 206739dcfe2e4cfc96e5e8d962d0575f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731357729038 2024-11-11T20:42:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T20:42:12,501 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:12,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:42:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-11T20:42:12,503 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:42:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T20:42:12,504 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:42:12,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:42:12,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742450_1626 (size=31058) 2024-11-11T20:42:12,510 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#C#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:12,511 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/f083151a603647f280825390ba4fabc7 is 50, key is test_row_0/C:col10/1731357731192/Put/seqid=0 2024-11-11T20:42:12,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742451_1627 (size=12104) 2024-11-11T20:42:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T20:42:12,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T20:42:12,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:12,656 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T20:42:12,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d50eb0d630c54413a55c7275ca5fde1b_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357731246/Put/seqid=0 2024-11-11T20:42:12,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742452_1628 (size=12154) 2024-11-11T20:42:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T20:42:12,909 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/e7865388b39644b898f796f23b7b64ad as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/e7865388b39644b898f796f23b7b64ad 2024-11-11T20:42:12,912 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/A of fe8aa7c45afd5cfd9441128b14c2ef12 into e7865388b39644b898f796f23b7b64ad(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:12,912 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:12,912 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/A, priority=13, startTime=1731357732474; duration=0sec 2024-11-11T20:42:12,912 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:12,912 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:A 2024-11-11T20:42:12,918 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/f083151a603647f280825390ba4fabc7 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f083151a603647f280825390ba4fabc7 2024-11-11T20:42:12,921 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/C of fe8aa7c45afd5cfd9441128b14c2ef12 into f083151a603647f280825390ba4fabc7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:12,921 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:12,921 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/C, priority=13, startTime=1731357732474; duration=0sec 2024-11-11T20:42:12,921 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:12,921 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:C 2024-11-11T20:42:13,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:13,067 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d50eb0d630c54413a55c7275ca5fde1b_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d50eb0d630c54413a55c7275ca5fde1b_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:13,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aed67c58856741e1826fca42a21031db, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:13,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aed67c58856741e1826fca42a21031db is 175, key is test_row_0/A:col10/1731357731246/Put/seqid=0 2024-11-11T20:42:13,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742453_1629 (size=30955) 2024-11-11T20:42:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T20:42:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:13,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357793410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,414 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in fe8aa7c45afd5cfd9441128b14c2ef12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:42:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357793412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357793413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357793413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357793414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,472 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aed67c58856741e1826fca42a21031db 2024-11-11T20:42:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/106fba8e1c0e4ccc93c0fce60719ac0e is 50, key is test_row_0/B:col10/1731357731246/Put/seqid=0 2024-11-11T20:42:13,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742454_1630 (size=12001) 2024-11-11T20:42:13,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357793520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357793520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357793521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357793521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T20:42:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357793730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357793731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357793731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357793731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:13,880 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/106fba8e1c0e4ccc93c0fce60719ac0e 2024-11-11T20:42:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/b51f8d0df4e741e588c530d60eb1cb9f is 50, key is test_row_0/C:col10/1731357731246/Put/seqid=0 2024-11-11T20:42:13,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742455_1631 (size=12001) 2024-11-11T20:42:14,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357794040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357794040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357794040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357794040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,291 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/b51f8d0df4e741e588c530d60eb1cb9f 2024-11-11T20:42:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aed67c58856741e1826fca42a21031db as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aed67c58856741e1826fca42a21031db 2024-11-11T20:42:14,298 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aed67c58856741e1826fca42a21031db, entries=150, sequenceid=79, filesize=30.2 K 2024-11-11T20:42:14,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/106fba8e1c0e4ccc93c0fce60719ac0e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/106fba8e1c0e4ccc93c0fce60719ac0e 2024-11-11T20:42:14,302 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/106fba8e1c0e4ccc93c0fce60719ac0e, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T20:42:14,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/b51f8d0df4e741e588c530d60eb1cb9f as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/b51f8d0df4e741e588c530d60eb1cb9f 2024-11-11T20:42:14,310 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/b51f8d0df4e741e588c530d60eb1cb9f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T20:42:14,311 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1654ms, sequenceid=79, compaction requested=false 2024-11-11T20:42:14,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:14,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:14,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-11T20:42:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-11T20:42:14,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-11T20:42:14,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8080 sec 2024-11-11T20:42:14,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.8120 sec 2024-11-11T20:42:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:14,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:42:14,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T20:42:14,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111117b88dafd8694113af9af0aa6eef08b4_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357733413/Put/seqid=0 2024-11-11T20:42:14,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357794572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357794574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357794576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357794576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742456_1632 (size=12154) 2024-11-11T20:42:14,584 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:14,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111117b88dafd8694113af9af0aa6eef08b4_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117b88dafd8694113af9af0aa6eef08b4_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:14,587 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/ec019a6e93104ff0be3eaf328bdad228, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:14,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/ec019a6e93104ff0be3eaf328bdad228 is 175, key is test_row_0/A:col10/1731357733413/Put/seqid=0 2024-11-11T20:42:14,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742457_1633 (size=30955) 2024-11-11T20:42:14,591 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/ec019a6e93104ff0be3eaf328bdad228 2024-11-11T20:42:14,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/4027951d14fd42b0859b2e4270937670 is 50, key is test_row_0/B:col10/1731357733413/Put/seqid=0 2024-11-11T20:42:14,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742458_1634 (size=12001) 2024-11-11T20:42:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T20:42:14,660 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:14,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:42:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-11T20:42:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T20:42:14,662 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:42:14,663 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:42:14,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:42:14,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357794680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357794681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357794691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357794691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T20:42:14,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T20:42:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:14,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:14,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357794890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357794891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357794901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357794901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:14,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T20:42:14,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:14,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T20:42:15,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/4027951d14fd42b0859b2e4270937670 2024-11-11T20:42:15,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/88b2cc9b8205491a8e502c37cfe7013b is 50, key is test_row_0/C:col10/1731357733413/Put/seqid=0 2024-11-11T20:42:15,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742459_1635 (size=12001) 2024-11-11T20:42:15,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T20:42:15,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:15,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:15,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:15,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357795201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357795201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357795210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357795210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T20:42:15,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T20:42:15,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/88b2cc9b8205491a8e502c37cfe7013b 2024-11-11T20:42:15,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/ec019a6e93104ff0be3eaf328bdad228 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ec019a6e93104ff0be3eaf328bdad228 2024-11-11T20:42:15,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ec019a6e93104ff0be3eaf328bdad228, entries=150, sequenceid=98, filesize=30.2 K 2024-11-11T20:42:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/4027951d14fd42b0859b2e4270937670 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/4027951d14fd42b0859b2e4270937670 2024-11-11T20:42:15,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/4027951d14fd42b0859b2e4270937670, entries=150, sequenceid=98, filesize=11.7 K 2024-11-11T20:42:15,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/88b2cc9b8205491a8e502c37cfe7013b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/88b2cc9b8205491a8e502c37cfe7013b 2024-11-11T20:42:15,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357795421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T20:42:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:15,425 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in fe8aa7c45afd5cfd9441128b14c2ef12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:42:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:15,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/88b2cc9b8205491a8e502c37cfe7013b, entries=150, sequenceid=98, filesize=11.7 K 2024-11-11T20:42:15,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for fe8aa7c45afd5cfd9441128b14c2ef12 in 873ms, sequenceid=98, compaction requested=true 2024-11-11T20:42:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:42:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:15,428 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:42:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:15,428 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:42:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/A is initiating minor compaction (all files) 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/B is initiating minor compaction (all files) 2024-11-11T20:42:15,429 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/A in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,429 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/B in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,429 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/e7865388b39644b898f796f23b7b64ad, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aed67c58856741e1826fca42a21031db, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ec019a6e93104ff0be3eaf328bdad228] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=90.8 K 2024-11-11T20:42:15,429 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/17e42c93c5474fc1a33ac19d30b22639, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/106fba8e1c0e4ccc93c0fce60719ac0e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/4027951d14fd42b0859b2e4270937670] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=35.3 K 2024-11-11T20:42:15,429 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/e7865388b39644b898f796f23b7b64ad, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aed67c58856741e1826fca42a21031db, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ec019a6e93104ff0be3eaf328bdad228] 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 17e42c93c5474fc1a33ac19d30b22639, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731357729038 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7865388b39644b898f796f23b7b64ad, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731357729038 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 106fba8e1c0e4ccc93c0fce60719ac0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731357731243 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting aed67c58856741e1826fca42a21031db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731357731243 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4027951d14fd42b0859b2e4270937670, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731357733407 2024-11-11T20:42:15,429 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec019a6e93104ff0be3eaf328bdad228, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731357733407 2024-11-11T20:42:15,433 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:15,434 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#B#compaction#540 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:15,434 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/ca5486b992114c23a05f9393e98c8af3 is 50, key is test_row_0/B:col10/1731357733413/Put/seqid=0 2024-11-11T20:42:15,435 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411114e712f489edf4b5b8824128ea3963032_fe8aa7c45afd5cfd9441128b14c2ef12 store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:15,436 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411114e712f489edf4b5b8824128ea3963032_fe8aa7c45afd5cfd9441128b14c2ef12, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:15,436 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114e712f489edf4b5b8824128ea3963032_fe8aa7c45afd5cfd9441128b14c2ef12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:15,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742460_1636 (size=4469) 2024-11-11T20:42:15,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742461_1637 (size=12207) 2024-11-11T20:42:15,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T20:42:15,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,578 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-11T20:42:15,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111609bd370f6ec43f7874583dd4e2a17e3_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357734575/Put/seqid=0 2024-11-11T20:42:15,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742462_1638 (size=12154) 2024-11-11T20:42:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:15,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357795729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357795730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357795732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357795735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T20:42:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357795841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357795841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357795841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357795841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:15,849 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#A#compaction#541 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:15,850 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/4fc5e1682137406a855ac917a6a83c78 is 175, key is test_row_0/A:col10/1731357733413/Put/seqid=0 2024-11-11T20:42:15,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742463_1639 (size=31161) 2024-11-11T20:42:15,854 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/ca5486b992114c23a05f9393e98c8af3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ca5486b992114c23a05f9393e98c8af3 2024-11-11T20:42:15,858 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/B of fe8aa7c45afd5cfd9441128b14c2ef12 into ca5486b992114c23a05f9393e98c8af3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:15,858 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:15,858 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/B, priority=13, startTime=1731357735428; duration=0sec 2024-11-11T20:42:15,858 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:15,858 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:B 2024-11-11T20:42:15,858 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:15,862 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/4fc5e1682137406a855ac917a6a83c78 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4fc5e1682137406a855ac917a6a83c78 2024-11-11T20:42:15,862 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:15,862 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/C is initiating minor compaction (all files) 2024-11-11T20:42:15,862 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/C in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:15,862 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f083151a603647f280825390ba4fabc7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/b51f8d0df4e741e588c530d60eb1cb9f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/88b2cc9b8205491a8e502c37cfe7013b] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=35.3 K 2024-11-11T20:42:15,863 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting f083151a603647f280825390ba4fabc7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731357729038 2024-11-11T20:42:15,863 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting b51f8d0df4e741e588c530d60eb1cb9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731357731243 2024-11-11T20:42:15,863 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b2cc9b8205491a8e502c37cfe7013b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731357733407 2024-11-11T20:42:15,866 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/A of fe8aa7c45afd5cfd9441128b14c2ef12 into 4fc5e1682137406a855ac917a6a83c78(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:15,866 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:15,866 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/A, priority=13, startTime=1731357735428; duration=0sec 2024-11-11T20:42:15,867 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:15,867 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:A 2024-11-11T20:42:15,871 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#C#compaction#543 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:15,872 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/c2727ab4f6b943c28338790b7bbc9550 is 50, key is test_row_0/C:col10/1731357733413/Put/seqid=0 2024-11-11T20:42:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742464_1640 (size=12207) 2024-11-11T20:42:15,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:15,989 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111609bd370f6ec43f7874583dd4e2a17e3_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111609bd370f6ec43f7874583dd4e2a17e3_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:15,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/a03334806e1b4990be24fe5127d04994, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:15,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/a03334806e1b4990be24fe5127d04994 is 175, key is test_row_0/A:col10/1731357734575/Put/seqid=0 2024-11-11T20:42:15,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742465_1641 (size=30955) 2024-11-11T20:42:16,009 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/a03334806e1b4990be24fe5127d04994 2024-11-11T20:42:16,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/0166d7125113435998a81c2b21fdb30e is 50, key is test_row_0/B:col10/1731357734575/Put/seqid=0 2024-11-11T20:42:16,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742466_1642 (size=12001) 2024-11-11T20:42:16,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357796051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357796051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357796051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357796051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,301 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/c2727ab4f6b943c28338790b7bbc9550 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c2727ab4f6b943c28338790b7bbc9550 2024-11-11T20:42:16,304 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/C of fe8aa7c45afd5cfd9441128b14c2ef12 into c2727ab4f6b943c28338790b7bbc9550(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:16,304 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:16,304 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/C, priority=13, startTime=1731357735428; duration=0sec 2024-11-11T20:42:16,305 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:16,305 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:C 2024-11-11T20:42:16,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357796361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357796361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357796361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357796361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,419 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/0166d7125113435998a81c2b21fdb30e 2024-11-11T20:42:16,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/8776b9d84a0d489994ae95c95312e45c is 50, key is test_row_0/C:col10/1731357734575/Put/seqid=0 2024-11-11T20:42:16,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742467_1643 (size=12001) 2024-11-11T20:42:16,430 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/8776b9d84a0d489994ae95c95312e45c 2024-11-11T20:42:16,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/a03334806e1b4990be24fe5127d04994 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a03334806e1b4990be24fe5127d04994 2024-11-11T20:42:16,436 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a03334806e1b4990be24fe5127d04994, entries=150, sequenceid=117, filesize=30.2 K 2024-11-11T20:42:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/0166d7125113435998a81c2b21fdb30e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/0166d7125113435998a81c2b21fdb30e 2024-11-11T20:42:16,439 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/0166d7125113435998a81c2b21fdb30e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T20:42:16,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/8776b9d84a0d489994ae95c95312e45c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8776b9d84a0d489994ae95c95312e45c 2024-11-11T20:42:16,443 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8776b9d84a0d489994ae95c95312e45c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T20:42:16,443 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for fe8aa7c45afd5cfd9441128b14c2ef12 in 866ms, sequenceid=117, compaction requested=false 2024-11-11T20:42:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-11T20:42:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-11T20:42:16,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-11T20:42:16,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-11-11T20:42:16,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.7870 sec 2024-11-11T20:42:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T20:42:16,811 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:16,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:42:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-11T20:42:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T20:42:16,813 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:42:16,813 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:42:16,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:42:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:16,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:42:16,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-11T20:42:16,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115379c9664a4f40cea519a682fecdfb73_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357735731/Put/seqid=0 2024-11-11T20:42:16,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742468_1644 (size=12304) 2024-11-11T20:42:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T20:42:16,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357796921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357796921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357796921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357796922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:16,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-11T20:42:16,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:16,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:16,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:16,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:16,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:17,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357797030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357797030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357797030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357797030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-11T20:42:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:17,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T20:42:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357797240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357797240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357797241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357797241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-11T20:42:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:17,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:17,302 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:17,305 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115379c9664a4f40cea519a682fecdfb73_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115379c9664a4f40cea519a682fecdfb73_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:17,307 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aa1d8f1fb73543dea67e79a5fe4bb310, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:17,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aa1d8f1fb73543dea67e79a5fe4bb310 is 175, key is test_row_0/A:col10/1731357735731/Put/seqid=0 2024-11-11T20:42:17,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742469_1645 (size=31105) 2024-11-11T20:42:17,331 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aa1d8f1fb73543dea67e79a5fe4bb310 2024-11-11T20:42:17,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/e4f38fe26c144e5dbc4697aa39f6e7af is 50, key is test_row_0/B:col10/1731357735731/Put/seqid=0 2024-11-11T20:42:17,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742470_1646 (size=12151) 2024-11-11T20:42:17,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/e4f38fe26c144e5dbc4697aa39f6e7af 2024-11-11T20:42:17,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/1e15f927a2cb4dcf9cbebedfb8502248 is 50, key is test_row_0/C:col10/1731357735731/Put/seqid=0 2024-11-11T20:42:17,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742471_1647 (size=12151) 2024-11-11T20:42:17,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/1e15f927a2cb4dcf9cbebedfb8502248 2024-11-11T20:42:17,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aa1d8f1fb73543dea67e79a5fe4bb310 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa1d8f1fb73543dea67e79a5fe4bb310 2024-11-11T20:42:17,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa1d8f1fb73543dea67e79a5fe4bb310, entries=150, sequenceid=139, filesize=30.4 K 2024-11-11T20:42:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/e4f38fe26c144e5dbc4697aa39f6e7af as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/e4f38fe26c144e5dbc4697aa39f6e7af 2024-11-11T20:42:17,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/e4f38fe26c144e5dbc4697aa39f6e7af, entries=150, sequenceid=139, filesize=11.9 K 2024-11-11T20:42:17,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/1e15f927a2cb4dcf9cbebedfb8502248 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/1e15f927a2cb4dcf9cbebedfb8502248 2024-11-11T20:42:17,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/1e15f927a2cb4dcf9cbebedfb8502248, entries=150, sequenceid=139, filesize=11.9 K 2024-11-11T20:42:17,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for fe8aa7c45afd5cfd9441128b14c2ef12 in 535ms, sequenceid=139, compaction requested=true 2024-11-11T20:42:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:42:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:17,410 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:17,410 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:42:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:42:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:17,411 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:17,411 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/A is initiating minor compaction (all files) 2024-11-11T20:42:17,411 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/A in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:17,411 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:17,411 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/B is initiating minor compaction (all files) 2024-11-11T20:42:17,411 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4fc5e1682137406a855ac917a6a83c78, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a03334806e1b4990be24fe5127d04994, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa1d8f1fb73543dea67e79a5fe4bb310] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=91.0 K 2024-11-11T20:42:17,411 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/B in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:17,411 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:17,411 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4fc5e1682137406a855ac917a6a83c78, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a03334806e1b4990be24fe5127d04994, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa1d8f1fb73543dea67e79a5fe4bb310] 2024-11-11T20:42:17,411 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ca5486b992114c23a05f9393e98c8af3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/0166d7125113435998a81c2b21fdb30e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/e4f38fe26c144e5dbc4697aa39f6e7af] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=35.5 K 2024-11-11T20:42:17,412 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ca5486b992114c23a05f9393e98c8af3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731357733407 2024-11-11T20:42:17,412 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fc5e1682137406a855ac917a6a83c78, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731357733407 2024-11-11T20:42:17,412 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0166d7125113435998a81c2b21fdb30e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731357734573 2024-11-11T20:42:17,412 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a03334806e1b4990be24fe5127d04994, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731357734573 2024-11-11T20:42:17,413 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa1d8f1fb73543dea67e79a5fe4bb310, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731357735731 2024-11-11T20:42:17,413 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f38fe26c144e5dbc4697aa39f6e7af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731357735731 2024-11-11T20:42:17,421 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:17,422 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#B#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:17,422 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/1f43acfa1e18497b9195654752b85710 is 50, key is test_row_0/B:col10/1731357735731/Put/seqid=0 2024-11-11T20:42:17,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-11T20:42:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:17,424 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T20:42:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T20:42:17,442 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111db9952c4891d44338dc1ae8f42cee83f_fe8aa7c45afd5cfd9441128b14c2ef12 store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:17,444 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111db9952c4891d44338dc1ae8f42cee83f_fe8aa7c45afd5cfd9441128b14c2ef12, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:17,444 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111db9952c4891d44338dc1ae8f42cee83f_fe8aa7c45afd5cfd9441128b14c2ef12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:17,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111199915383a2604090b35e231352ed4297_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357736890/Put/seqid=0 2024-11-11T20:42:17,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742472_1648 (size=12459) 2024-11-11T20:42:17,461 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/1f43acfa1e18497b9195654752b85710 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/1f43acfa1e18497b9195654752b85710 2024-11-11T20:42:17,466 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/B of fe8aa7c45afd5cfd9441128b14c2ef12 into 1f43acfa1e18497b9195654752b85710(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:17,466 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:17,466 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/B, priority=13, startTime=1731357737410; duration=0sec 2024-11-11T20:42:17,466 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:17,466 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:B 2024-11-11T20:42:17,466 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:17,467 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:17,467 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/C is initiating minor compaction (all files) 2024-11-11T20:42:17,467 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/C in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:17,467 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c2727ab4f6b943c28338790b7bbc9550, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8776b9d84a0d489994ae95c95312e45c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/1e15f927a2cb4dcf9cbebedfb8502248] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=35.5 K 2024-11-11T20:42:17,467 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c2727ab4f6b943c28338790b7bbc9550, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731357733407 2024-11-11T20:42:17,467 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8776b9d84a0d489994ae95c95312e45c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731357734573 2024-11-11T20:42:17,467 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e15f927a2cb4dcf9cbebedfb8502248, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731357735731 2024-11-11T20:42:17,474 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#C#compaction#552 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:17,474 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/49ec8848f5aa45869ab65fbb3a046c88 is 50, key is test_row_0/C:col10/1731357735731/Put/seqid=0 2024-11-11T20:42:17,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742473_1649 (size=4469) 2024-11-11T20:42:17,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742474_1650 (size=12304) 2024-11-11T20:42:17,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742475_1651 (size=12459) 2024-11-11T20:42:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:17,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:17,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357797566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357797567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357797568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357797569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357797681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357797681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357797681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357797682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357797891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357797891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357797891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357797891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:17,895 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#A#compaction#550 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:17,895 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/687994d4190745c4821d174c50c9de45 is 175, key is test_row_0/A:col10/1731357735731/Put/seqid=0 2024-11-11T20:42:17,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742476_1652 (size=31413) 2024-11-11T20:42:17,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:17,901 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111199915383a2604090b35e231352ed4297_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111199915383a2604090b35e231352ed4297_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:17,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/00fa5e4e130b4859ae2a24f873aff714, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/00fa5e4e130b4859ae2a24f873aff714 is 175, key is test_row_0/A:col10/1731357736890/Put/seqid=0 2024-11-11T20:42:17,907 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/49ec8848f5aa45869ab65fbb3a046c88 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/49ec8848f5aa45869ab65fbb3a046c88 2024-11-11T20:42:17,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742477_1653 (size=31105) 2024-11-11T20:42:17,909 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/00fa5e4e130b4859ae2a24f873aff714 2024-11-11T20:42:17,912 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/C of fe8aa7c45afd5cfd9441128b14c2ef12 into 49ec8848f5aa45869ab65fbb3a046c88(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:17,912 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:17,912 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/C, priority=13, startTime=1731357737410; duration=0sec 2024-11-11T20:42:17,912 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:17,912 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:C 2024-11-11T20:42:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/37eb546cf5674ba78ec855f314f0b911 is 50, key is test_row_0/B:col10/1731357736890/Put/seqid=0 2024-11-11T20:42:17,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742478_1654 (size=12151) 2024-11-11T20:42:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T20:42:18,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357798200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357798200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357798200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357798201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,301 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/687994d4190745c4821d174c50c9de45 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/687994d4190745c4821d174c50c9de45 2024-11-11T20:42:18,304 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/A of fe8aa7c45afd5cfd9441128b14c2ef12 into 687994d4190745c4821d174c50c9de45(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:18,304 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:18,304 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/A, priority=13, startTime=1731357737410; duration=0sec 2024-11-11T20:42:18,304 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:18,305 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:A 2024-11-11T20:42:18,317 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/37eb546cf5674ba78ec855f314f0b911 2024-11-11T20:42:18,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/33240291e0b64b2ca2e805cd8fb6e04b is 50, key is test_row_0/C:col10/1731357736890/Put/seqid=0 2024-11-11T20:42:18,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742479_1655 (size=12151) 2024-11-11T20:42:18,328 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/33240291e0b64b2ca2e805cd8fb6e04b 2024-11-11T20:42:18,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/00fa5e4e130b4859ae2a24f873aff714 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/00fa5e4e130b4859ae2a24f873aff714 2024-11-11T20:42:18,334 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/00fa5e4e130b4859ae2a24f873aff714, entries=150, sequenceid=156, filesize=30.4 K 2024-11-11T20:42:18,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/37eb546cf5674ba78ec855f314f0b911 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/37eb546cf5674ba78ec855f314f0b911 2024-11-11T20:42:18,337 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/37eb546cf5674ba78ec855f314f0b911, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T20:42:18,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/33240291e0b64b2ca2e805cd8fb6e04b as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/33240291e0b64b2ca2e805cd8fb6e04b 2024-11-11T20:42:18,340 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/33240291e0b64b2ca2e805cd8fb6e04b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T20:42:18,340 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for fe8aa7c45afd5cfd9441128b14c2ef12 in 916ms, sequenceid=156, compaction requested=false 2024-11-11T20:42:18,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:18,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:18,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-11T20:42:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-11T20:42:18,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-11T20:42:18,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5290 sec 2024-11-11T20:42:18,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.5330 sec 2024-11-11T20:42:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:18,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:42:18,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T20:42:18,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115a2988683da44527b69920842f7713b5_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357738713/Put/seqid=0 2024-11-11T20:42:18,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742480_1656 (size=17284) 2024-11-11T20:42:18,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357798726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357798727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357798730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357798731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357798841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357798841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357798841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357798841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T20:42:18,961 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:18,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:42:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-11T20:42:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T20:42:18,963 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:42:18,963 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:42:18,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:42:19,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357799051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357799051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357799051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357799051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T20:42:19,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T20:42:19,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:19,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,122 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:19,125 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115a2988683da44527b69920842f7713b5_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115a2988683da44527b69920842f7713b5_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:19,126 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/bb3ca50d349043ff82b388ba62bf3d0e, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:19,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/bb3ca50d349043ff82b388ba62bf3d0e is 175, key is test_row_0/A:col10/1731357738713/Put/seqid=0 2024-11-11T20:42:19,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742481_1657 (size=48389) 2024-11-11T20:42:19,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T20:42:19,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:19,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T20:42:19,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357799361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357799361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357799361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357799361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T20:42:19,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:19,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37304 deadline: 1731357799441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,445 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_1' in fe8aa7c45afd5cfd9441128b14c2ef12 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:42:19,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,529 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/bb3ca50d349043ff82b388ba62bf3d0e 2024-11-11T20:42:19,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/5a09d9d7d5864c01bdc2c26893dcf51a is 50, key is test_row_0/B:col10/1731357738713/Put/seqid=0 2024-11-11T20:42:19,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742482_1658 (size=12151) 2024-11-11T20:42:19,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T20:42:19,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:19,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T20:42:19,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T20:42:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357799871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357799871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357799871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357799871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:19,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T20:42:19,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:19,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:19,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/5a09d9d7d5864c01bdc2c26893dcf51a 2024-11-11T20:42:19,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/c0d0163fe19049a181235e2bf38e3b91 is 50, key is test_row_0/C:col10/1731357738713/Put/seqid=0 2024-11-11T20:42:19,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742483_1659 (size=12151) 2024-11-11T20:42:19,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/c0d0163fe19049a181235e2bf38e3b91 2024-11-11T20:42:19,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/bb3ca50d349043ff82b388ba62bf3d0e as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/bb3ca50d349043ff82b388ba62bf3d0e 2024-11-11T20:42:19,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/bb3ca50d349043ff82b388ba62bf3d0e, entries=250, sequenceid=182, filesize=47.3 K 2024-11-11T20:42:19,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/5a09d9d7d5864c01bdc2c26893dcf51a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/5a09d9d7d5864c01bdc2c26893dcf51a 2024-11-11T20:42:19,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/5a09d9d7d5864c01bdc2c26893dcf51a, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T20:42:19,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/c0d0163fe19049a181235e2bf38e3b91 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c0d0163fe19049a181235e2bf38e3b91 2024-11-11T20:42:19,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c0d0163fe19049a181235e2bf38e3b91, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T20:42:19,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1246ms, sequenceid=182, compaction requested=true 2024-11-11T20:42:19,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:42:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:19,961 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:42:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:42:19,961 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:19,962 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:19,962 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:19,963 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/B is initiating minor compaction (all files) 2024-11-11T20:42:19,963 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/A is initiating minor compaction (all files) 2024-11-11T20:42:19,963 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/B in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,963 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/A in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,963 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/687994d4190745c4821d174c50c9de45, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/00fa5e4e130b4859ae2a24f873aff714, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/bb3ca50d349043ff82b388ba62bf3d0e] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=108.3 K 2024-11-11T20:42:19,963 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/1f43acfa1e18497b9195654752b85710, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/37eb546cf5674ba78ec855f314f0b911, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/5a09d9d7d5864c01bdc2c26893dcf51a] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=35.9 K 2024-11-11T20:42:19,963 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:19,963 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/687994d4190745c4821d174c50c9de45, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/00fa5e4e130b4859ae2a24f873aff714, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/bb3ca50d349043ff82b388ba62bf3d0e] 2024-11-11T20:42:19,963 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f43acfa1e18497b9195654752b85710, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731357735731 2024-11-11T20:42:19,963 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 687994d4190745c4821d174c50c9de45, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731357735731 2024-11-11T20:42:19,963 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00fa5e4e130b4859ae2a24f873aff714, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731357736879 2024-11-11T20:42:19,963 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 37eb546cf5674ba78ec855f314f0b911, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731357736879 2024-11-11T20:42:19,964 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb3ca50d349043ff82b388ba62bf3d0e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731357737568 2024-11-11T20:42:19,964 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a09d9d7d5864c01bdc2c26893dcf51a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731357737568 2024-11-11T20:42:19,968 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:19,969 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#B#compaction#559 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:19,970 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/ffafb8d4ba614ad28aabd51e96eb8fea is 50, key is test_row_0/B:col10/1731357738713/Put/seqid=0 2024-11-11T20:42:19,970 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111960de0f251a14b64a266bd6f1e836b8f_fe8aa7c45afd5cfd9441128b14c2ef12 store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:19,972 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111960de0f251a14b64a266bd6f1e836b8f_fe8aa7c45afd5cfd9441128b14c2ef12, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:19,972 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111960de0f251a14b64a266bd6f1e836b8f_fe8aa7c45afd5cfd9441128b14c2ef12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:19,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742484_1660 (size=12561) 2024-11-11T20:42:19,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742485_1661 (size=4469) 2024-11-11T20:42:20,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T20:42:20,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:20,031 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T20:42:20,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e2e0e360996d4e518cee96f50f81f5a7_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357738725/Put/seqid=0 2024-11-11T20:42:20,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742486_1662 (size=12304) 2024-11-11T20:42:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T20:42:20,381 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#A#compaction#558 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:20,382 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/c22a499bb1ed4280a62fcdde1ecb4b70 is 175, key is test_row_0/A:col10/1731357738713/Put/seqid=0 2024-11-11T20:42:20,384 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/ffafb8d4ba614ad28aabd51e96eb8fea as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ffafb8d4ba614ad28aabd51e96eb8fea 2024-11-11T20:42:20,388 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/B of fe8aa7c45afd5cfd9441128b14c2ef12 into ffafb8d4ba614ad28aabd51e96eb8fea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:20,388 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:20,388 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/B, priority=13, startTime=1731357739961; duration=0sec 2024-11-11T20:42:20,388 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:20,388 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:B 2024-11-11T20:42:20,388 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:20,389 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:20,389 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/C is initiating minor compaction (all files) 2024-11-11T20:42:20,389 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/C in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:20,389 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/49ec8848f5aa45869ab65fbb3a046c88, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/33240291e0b64b2ca2e805cd8fb6e04b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c0d0163fe19049a181235e2bf38e3b91] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=35.9 K 2024-11-11T20:42:20,389 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 49ec8848f5aa45869ab65fbb3a046c88, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731357735731 2024-11-11T20:42:20,390 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 33240291e0b64b2ca2e805cd8fb6e04b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731357736879 2024-11-11T20:42:20,390 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d0163fe19049a181235e2bf38e3b91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731357737568 2024-11-11T20:42:20,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742487_1663 (size=31515) 2024-11-11T20:42:20,398 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#C#compaction#561 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:20,398 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/edd6558f51bb432b9cc23512b6d4e552 is 50, key is test_row_0/C:col10/1731357738713/Put/seqid=0 2024-11-11T20:42:20,402 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/c22a499bb1ed4280a62fcdde1ecb4b70 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/c22a499bb1ed4280a62fcdde1ecb4b70 2024-11-11T20:42:20,413 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/A of fe8aa7c45afd5cfd9441128b14c2ef12 into c22a499bb1ed4280a62fcdde1ecb4b70(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:20,413 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:20,413 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/A, priority=13, startTime=1731357739961; duration=0sec 2024-11-11T20:42:20,413 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:20,413 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:A 2024-11-11T20:42:20,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742488_1664 (size=12561) 2024-11-11T20:42:20,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:20,457 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e2e0e360996d4e518cee96f50f81f5a7_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e2e0e360996d4e518cee96f50f81f5a7_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:20,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/7e095bbd10f44c448abee7943ea46330, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:20,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/7e095bbd10f44c448abee7943ea46330 is 175, key is test_row_0/A:col10/1731357738725/Put/seqid=0 2024-11-11T20:42:20,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742489_1665 (size=31105) 2024-11-11T20:42:20,829 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/edd6558f51bb432b9cc23512b6d4e552 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/edd6558f51bb432b9cc23512b6d4e552 2024-11-11T20:42:20,836 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/C of fe8aa7c45afd5cfd9441128b14c2ef12 into edd6558f51bb432b9cc23512b6d4e552(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:20,836 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:20,836 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/C, priority=13, startTime=1731357739961; duration=0sec 2024-11-11T20:42:20,836 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:20,836 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:C 2024-11-11T20:42:20,871 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/7e095bbd10f44c448abee7943ea46330 2024-11-11T20:42:20,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/6f95abaa1d58401d9ca8ce6ca4ed8164 is 50, key is test_row_0/B:col10/1731357738725/Put/seqid=0 2024-11-11T20:42:20,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742490_1666 (size=12151) 2024-11-11T20:42:20,880 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/6f95abaa1d58401d9ca8ce6ca4ed8164 2024-11-11T20:42:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/706bd57358a245d59a571cb2c2b29c25 is 50, key is test_row_0/C:col10/1731357738725/Put/seqid=0 2024-11-11T20:42:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357800904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357800905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:20,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742491_1667 (size=12151) 2024-11-11T20:42:20,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357800910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:20,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357800910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357801021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357801021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357801021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357801021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T20:42:21,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357801231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357801231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357801231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357801231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,312 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/706bd57358a245d59a571cb2c2b29c25 2024-11-11T20:42:21,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/7e095bbd10f44c448abee7943ea46330 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/7e095bbd10f44c448abee7943ea46330 2024-11-11T20:42:21,318 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/7e095bbd10f44c448abee7943ea46330, entries=150, sequenceid=196, filesize=30.4 K 2024-11-11T20:42:21,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/6f95abaa1d58401d9ca8ce6ca4ed8164 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/6f95abaa1d58401d9ca8ce6ca4ed8164 2024-11-11T20:42:21,321 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/6f95abaa1d58401d9ca8ce6ca4ed8164, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T20:42:21,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/706bd57358a245d59a571cb2c2b29c25 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/706bd57358a245d59a571cb2c2b29c25 2024-11-11T20:42:21,324 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/706bd57358a245d59a571cb2c2b29c25, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T20:42:21,325 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1295ms, sequenceid=196, compaction requested=false 2024-11-11T20:42:21,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:21,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:21,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-11T20:42:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-11T20:42:21,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-11T20:42:21,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3630 sec 2024-11-11T20:42:21,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 2.3660 sec 2024-11-11T20:42:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:21,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:42:21,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:42:21,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d3c44bd3dd654623842f2b9755d37eba_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357740899/Put/seqid=0 2024-11-11T20:42:21,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742492_1668 (size=12304) 2024-11-11T20:42:21,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357801549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357801550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357801552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357801554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357801661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357801661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357801661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357801661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357801870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357801871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357801871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357801871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:21,951 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:21,954 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d3c44bd3dd654623842f2b9755d37eba_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d3c44bd3dd654623842f2b9755d37eba_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:21,955 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aa236d6c68e444949255861149ab30a6, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:21,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aa236d6c68e444949255861149ab30a6 is 175, key is test_row_0/A:col10/1731357740899/Put/seqid=0 2024-11-11T20:42:21,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742493_1669 (size=31105) 2024-11-11T20:42:22,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357802180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357802180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:22,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357802180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357802180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:22,358 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aa236d6c68e444949255861149ab30a6 2024-11-11T20:42:22,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/2195a3e349ea454f9cfe49274a26d6b5 is 50, key is test_row_0/B:col10/1731357740899/Put/seqid=0 2024-11-11T20:42:22,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742494_1670 (size=12151) 2024-11-11T20:42:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357802690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357802690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357802690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357802690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:22,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/2195a3e349ea454f9cfe49274a26d6b5 2024-11-11T20:42:22,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/8a6a064126174c0fa52b7329657f999d is 50, key is test_row_0/C:col10/1731357740899/Put/seqid=0 2024-11-11T20:42:22,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742495_1671 (size=12151) 2024-11-11T20:42:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T20:42:23,121 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:23,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:42:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-11T20:42:23,123 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:42:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T20:42:23,124 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:42:23,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:42:23,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/8a6a064126174c0fa52b7329657f999d 2024-11-11T20:42:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/aa236d6c68e444949255861149ab30a6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa236d6c68e444949255861149ab30a6 2024-11-11T20:42:23,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa236d6c68e444949255861149ab30a6, entries=150, sequenceid=223, filesize=30.4 K 2024-11-11T20:42:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/2195a3e349ea454f9cfe49274a26d6b5 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/2195a3e349ea454f9cfe49274a26d6b5 2024-11-11T20:42:23,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/2195a3e349ea454f9cfe49274a26d6b5, entries=150, sequenceid=223, filesize=11.9 K 2024-11-11T20:42:23,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/8a6a064126174c0fa52b7329657f999d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8a6a064126174c0fa52b7329657f999d 2024-11-11T20:42:23,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8a6a064126174c0fa52b7329657f999d, entries=150, sequenceid=223, filesize=11.9 K 2024-11-11T20:42:23,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1661ms, sequenceid=223, compaction requested=true 2024-11-11T20:42:23,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:23,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:42:23,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:23,205 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:23,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:42:23,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:23,205 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:23,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:42:23,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:23,205 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:23,205 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:23,205 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/B is initiating minor compaction (all files) 2024-11-11T20:42:23,205 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/A is initiating minor compaction (all files) 2024-11-11T20:42:23,205 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/B in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:23,205 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/A in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:23,205 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ffafb8d4ba614ad28aabd51e96eb8fea, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/6f95abaa1d58401d9ca8ce6ca4ed8164, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/2195a3e349ea454f9cfe49274a26d6b5] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=36.0 K 2024-11-11T20:42:23,205 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/c22a499bb1ed4280a62fcdde1ecb4b70, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/7e095bbd10f44c448abee7943ea46330, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa236d6c68e444949255861149ab30a6] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=91.5 K 2024-11-11T20:42:23,206 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:23,206 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/c22a499bb1ed4280a62fcdde1ecb4b70, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/7e095bbd10f44c448abee7943ea46330, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa236d6c68e444949255861149ab30a6] 2024-11-11T20:42:23,207 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting ffafb8d4ba614ad28aabd51e96eb8fea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731357737568 2024-11-11T20:42:23,207 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22a499bb1ed4280a62fcdde1ecb4b70, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731357737568 2024-11-11T20:42:23,207 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e095bbd10f44c448abee7943ea46330, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731357738720 2024-11-11T20:42:23,207 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f95abaa1d58401d9ca8ce6ca4ed8164, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731357738720 2024-11-11T20:42:23,207 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa236d6c68e444949255861149ab30a6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731357740899 2024-11-11T20:42:23,207 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2195a3e349ea454f9cfe49274a26d6b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731357740899 2024-11-11T20:42:23,213 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:23,214 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#B#compaction#567 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:23,214 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111fd39803eddd2425cb6028ae1483d0e7e_fe8aa7c45afd5cfd9441128b14c2ef12 store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:23,215 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/a9a92d53e4aa4b90b7d3387c25cf06b3 is 50, key is test_row_0/B:col10/1731357740899/Put/seqid=0 2024-11-11T20:42:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111fd39803eddd2425cb6028ae1483d0e7e_fe8aa7c45afd5cfd9441128b14c2ef12, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:23,216 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fd39803eddd2425cb6028ae1483d0e7e_fe8aa7c45afd5cfd9441128b14c2ef12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T20:42:23,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742497_1673 (size=4469) 2024-11-11T20:42:23,239 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#A#compaction#568 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:23,239 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/14a3ea6bfd66413fb408a8a7b0dbf7ee is 175, key is test_row_0/A:col10/1731357740899/Put/seqid=0 2024-11-11T20:42:23,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742496_1672 (size=12663) 2024-11-11T20:42:23,247 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/a9a92d53e4aa4b90b7d3387c25cf06b3 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/a9a92d53e4aa4b90b7d3387c25cf06b3 2024-11-11T20:42:23,250 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/B of fe8aa7c45afd5cfd9441128b14c2ef12 into a9a92d53e4aa4b90b7d3387c25cf06b3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:23,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:23,250 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/B, priority=13, startTime=1731357743205; duration=0sec 2024-11-11T20:42:23,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:23,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:B 2024-11-11T20:42:23,250 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:42:23,252 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:42:23,252 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/C is initiating minor compaction (all files) 2024-11-11T20:42:23,252 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/C in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:23,252 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/edd6558f51bb432b9cc23512b6d4e552, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/706bd57358a245d59a571cb2c2b29c25, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8a6a064126174c0fa52b7329657f999d] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=36.0 K 2024-11-11T20:42:23,252 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting edd6558f51bb432b9cc23512b6d4e552, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731357737568 2024-11-11T20:42:23,253 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 706bd57358a245d59a571cb2c2b29c25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731357738720 2024-11-11T20:42:23,253 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6a064126174c0fa52b7329657f999d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731357740899 2024-11-11T20:42:23,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742498_1674 (size=31617) 2024-11-11T20:42:23,262 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/14a3ea6bfd66413fb408a8a7b0dbf7ee as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/14a3ea6bfd66413fb408a8a7b0dbf7ee 2024-11-11T20:42:23,262 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#C#compaction#569 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:23,263 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/4baebe9cde9d431aa13f76bea7730eba is 50, key is test_row_0/C:col10/1731357740899/Put/seqid=0 2024-11-11T20:42:23,267 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/A of fe8aa7c45afd5cfd9441128b14c2ef12 into 14a3ea6bfd66413fb408a8a7b0dbf7ee(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:23,267 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:23,267 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/A, priority=13, startTime=1731357743205; duration=0sec 2024-11-11T20:42:23,267 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:23,267 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:A 2024-11-11T20:42:23,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742499_1675 (size=12663) 2024-11-11T20:42:23,275 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/4baebe9cde9d431aa13f76bea7730eba as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4baebe9cde9d431aa13f76bea7730eba 2024-11-11T20:42:23,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-11T20:42:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:23,277 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:42:23,278 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/C of fe8aa7c45afd5cfd9441128b14c2ef12 into 4baebe9cde9d431aa13f76bea7730eba(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:23,278 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:23,278 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/C, priority=13, startTime=1731357743205; duration=0sec 2024-11-11T20:42:23,279 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:23,279 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:C 2024-11-11T20:42:23,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111115dabb8b6d4c4608be7ed5395ab7cd34_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357741548/Put/seqid=0 2024-11-11T20:42:23,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742500_1676 (size=12304) 2024-11-11T20:42:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T20:42:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:23,687 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111115dabb8b6d4c4608be7ed5395ab7cd34_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111115dabb8b6d4c4608be7ed5395ab7cd34_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:23,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/f731abd190424e98a688d91e04a1a211, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:23,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/f731abd190424e98a688d91e04a1a211 is 175, key is test_row_0/A:col10/1731357741548/Put/seqid=0 2024-11-11T20:42:23,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742501_1677 (size=31105) 2024-11-11T20:42:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:23,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:23,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357803724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357803725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:23,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357803726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357803727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T20:42:23,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357803840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357803841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357803841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357803841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357804050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357804050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357804050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357804051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,091 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/f731abd190424e98a688d91e04a1a211 2024-11-11T20:42:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/c43f685242e24f15a4534774f820acdb is 50, key is test_row_0/B:col10/1731357741548/Put/seqid=0 2024-11-11T20:42:24,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742502_1678 (size=12151) 2024-11-11T20:42:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T20:42:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357804361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357804361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357804361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357804361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,501 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/c43f685242e24f15a4534774f820acdb 2024-11-11T20:42:24,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/4a30e9fbc0cc4100bec51fcf3b64a815 is 50, key is test_row_0/C:col10/1731357741548/Put/seqid=0 2024-11-11T20:42:24,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742503_1679 (size=12151) 2024-11-11T20:42:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357804871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357804871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357804871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357804871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:24,910 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/4a30e9fbc0cc4100bec51fcf3b64a815 2024-11-11T20:42:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/f731abd190424e98a688d91e04a1a211 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/f731abd190424e98a688d91e04a1a211 2024-11-11T20:42:24,916 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/f731abd190424e98a688d91e04a1a211, entries=150, sequenceid=237, filesize=30.4 K 2024-11-11T20:42:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/c43f685242e24f15a4534774f820acdb as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c43f685242e24f15a4534774f820acdb 2024-11-11T20:42:24,919 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c43f685242e24f15a4534774f820acdb, entries=150, sequenceid=237, filesize=11.9 K 2024-11-11T20:42:24,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/4a30e9fbc0cc4100bec51fcf3b64a815 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4a30e9fbc0cc4100bec51fcf3b64a815 2024-11-11T20:42:24,925 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4a30e9fbc0cc4100bec51fcf3b64a815, entries=150, sequenceid=237, filesize=11.9 K 2024-11-11T20:42:24,925 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1648ms, sequenceid=237, compaction requested=false 2024-11-11T20:42:24,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:24,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:24,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-11T20:42:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-11T20:42:24,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-11T20:42:24,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-11-11T20:42:24,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 1.8070 sec 2024-11-11T20:42:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T20:42:25,270 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:25,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:42:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-11-11T20:42:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T20:42:25,272 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:42:25,272 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:42:25,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:42:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T20:42:25,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-11T20:42:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:25,425 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T20:42:25,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e5f2a40d03d9468a9cab7d2b63b53ad0_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357743726/Put/seqid=0 2024-11-11T20:42:25,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742504_1680 (size=12404) 2024-11-11T20:42:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T20:42:25,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:25,837 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e5f2a40d03d9468a9cab7d2b63b53ad0_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e5f2a40d03d9468a9cab7d2b63b53ad0_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:25,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/2f187a62f7ad4ba697d45fc4a5a50a38, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:25,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/2f187a62f7ad4ba697d45fc4a5a50a38 is 175, key is test_row_0/A:col10/1731357743726/Put/seqid=0 2024-11-11T20:42:25,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742505_1681 (size=31205) 2024-11-11T20:42:25,842 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/2f187a62f7ad4ba697d45fc4a5a50a38 2024-11-11T20:42:25,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/9f4c9d3f882942878aa748a8bcbc1446 is 50, key is test_row_0/B:col10/1731357743726/Put/seqid=0 2024-11-11T20:42:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742506_1682 (size=12251) 2024-11-11T20:42:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:25,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:25,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:25,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357805889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357805890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:25,894 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_2' in fe8aa7c45afd5cfd9441128b14c2ef12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:42:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:25,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357805896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:25,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357805897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:25,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:25,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:25,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T20:42:26,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357806001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:26,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357806010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:26,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357806010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357806211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:26,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357806220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:26,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357806220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:26,255 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/9f4c9d3f882942878aa748a8bcbc1446 2024-11-11T20:42:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/03f4cd24e889469999153e735dd03032 is 50, key is test_row_0/C:col10/1731357743726/Put/seqid=0 2024-11-11T20:42:26,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742507_1683 (size=12251) 2024-11-11T20:42:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T20:42:26,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357806521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:26,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357806531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:26,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357806531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:26,673 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/03f4cd24e889469999153e735dd03032 2024-11-11T20:42:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/2f187a62f7ad4ba697d45fc4a5a50a38 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/2f187a62f7ad4ba697d45fc4a5a50a38 2024-11-11T20:42:26,680 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/2f187a62f7ad4ba697d45fc4a5a50a38, entries=150, sequenceid=263, filesize=30.5 K 2024-11-11T20:42:26,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/9f4c9d3f882942878aa748a8bcbc1446 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f4c9d3f882942878aa748a8bcbc1446 2024-11-11T20:42:26,689 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f4c9d3f882942878aa748a8bcbc1446, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T20:42:26,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/03f4cd24e889469999153e735dd03032 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/03f4cd24e889469999153e735dd03032 2024-11-11T20:42:26,693 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/03f4cd24e889469999153e735dd03032, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T20:42:26,693 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1268ms, sequenceid=263, compaction requested=true 2024-11-11T20:42:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=179 2024-11-11T20:42:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=179 2024-11-11T20:42:26,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-11T20:42:26,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-11-11T20:42:26,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 1.4260 sec 2024-11-11T20:42:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:27,033 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T20:42:27,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T20:42:27,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c0e85689625e44fba85a385008caea7f_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357745892/Put/seqid=0 2024-11-11T20:42:27,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742508_1684 (size=14994) 2024-11-11T20:42:27,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357807069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357807070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357807071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357807180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357807180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357807180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357807390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357807390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357807390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T20:42:27,421 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:27,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:42:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees 2024-11-11T20:42:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-11T20:42:27,422 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:42:27,423 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:42:27,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:42:27,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:27,444 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c0e85689625e44fba85a385008caea7f_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0e85689625e44fba85a385008caea7f_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:27,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/012b2aed3212408394946cd967fffa27, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:27,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/012b2aed3212408394946cd967fffa27 is 175, key is test_row_0/A:col10/1731357745892/Put/seqid=0 2024-11-11T20:42:27,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742509_1685 (size=39949) 2024-11-11T20:42:27,448 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/012b2aed3212408394946cd967fffa27 2024-11-11T20:42:27,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/099b615474624cdeaac07029b13d0d5c is 50, key is test_row_0/B:col10/1731357745892/Put/seqid=0 2024-11-11T20:42:27,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742510_1686 (size=12301) 2024-11-11T20:42:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-11T20:42:27,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-11T20:42:27,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:27,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:27,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:27,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:27,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:27,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357807701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357807701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357807701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:27,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-11T20:42:27,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:27,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:27,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:27,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:27,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-11T20:42:27,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/099b615474624cdeaac07029b13d0d5c 2024-11-11T20:42:27,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/2487fe12d6b14b578ecaeed268483d6c is 50, key is test_row_0/C:col10/1731357745892/Put/seqid=0 2024-11-11T20:42:27,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742511_1687 (size=12301) 2024-11-11T20:42:27,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-11T20:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:27,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:27,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:27,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1731357807911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:27,913 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44271,1731357576703 for 'test_row_2' in fe8aa7c45afd5cfd9441128b14c2ef12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:42:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:28,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-11T20:42:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:28,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-11T20:42:28,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-11T20:42:28,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:28,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:28,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:28,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:28,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:42:28,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37254 deadline: 1731357808210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:28,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:28,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:28,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:28,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731357808210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:28,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:42:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44271 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37296 deadline: 1731357808210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 , the old value is region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:42:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., hostname=51ca66f7ee3c,44271,1731357576703, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:42:28,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/2487fe12d6b14b578ecaeed268483d6c 2024-11-11T20:42:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/012b2aed3212408394946cd967fffa27 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012b2aed3212408394946cd967fffa27 2024-11-11T20:42:28,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012b2aed3212408394946cd967fffa27, entries=200, sequenceid=275, filesize=39.0 K 2024-11-11T20:42:28,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/099b615474624cdeaac07029b13d0d5c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/099b615474624cdeaac07029b13d0d5c 2024-11-11T20:42:28,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/099b615474624cdeaac07029b13d0d5c, entries=150, sequenceid=275, filesize=12.0 K 2024-11-11T20:42:28,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/2487fe12d6b14b578ecaeed268483d6c as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/2487fe12d6b14b578ecaeed268483d6c 2024-11-11T20:42:28,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/2487fe12d6b14b578ecaeed268483d6c, entries=150, sequenceid=275, filesize=12.0 K 2024-11-11T20:42:28,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1244ms, sequenceid=275, compaction requested=true 2024-11-11T20:42:28,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:28,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:42:28,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:28,277 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:42:28,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:42:28,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:28,277 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:42:28,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8aa7c45afd5cfd9441128b14c2ef12:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:42:28,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:28,278 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133876 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:42:28,278 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/A is initiating minor compaction (all files) 2024-11-11T20:42:28,278 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/A in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:28,278 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/14a3ea6bfd66413fb408a8a7b0dbf7ee, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/f731abd190424e98a688d91e04a1a211, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/2f187a62f7ad4ba697d45fc4a5a50a38, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012b2aed3212408394946cd967fffa27] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=130.7 K 2024-11-11T20:42:28,278 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:28,278 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. files: [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/14a3ea6bfd66413fb408a8a7b0dbf7ee, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/f731abd190424e98a688d91e04a1a211, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/2f187a62f7ad4ba697d45fc4a5a50a38, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012b2aed3212408394946cd967fffa27] 2024-11-11T20:42:28,279 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:42:28,279 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/B is initiating minor compaction (all files) 2024-11-11T20:42:28,279 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a3ea6bfd66413fb408a8a7b0dbf7ee, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731357740899 2024-11-11T20:42:28,279 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/B in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:28,279 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/a9a92d53e4aa4b90b7d3387c25cf06b3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c43f685242e24f15a4534774f820acdb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f4c9d3f882942878aa748a8bcbc1446, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/099b615474624cdeaac07029b13d0d5c] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=48.2 K 2024-11-11T20:42:28,279 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f731abd190424e98a688d91e04a1a211, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731357741548 2024-11-11T20:42:28,279 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a92d53e4aa4b90b7d3387c25cf06b3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731357740899 2024-11-11T20:42:28,279 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f187a62f7ad4ba697d45fc4a5a50a38, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731357743719 2024-11-11T20:42:28,279 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting c43f685242e24f15a4534774f820acdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731357741548 2024-11-11T20:42:28,280 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 012b2aed3212408394946cd967fffa27, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731357745883 2024-11-11T20:42:28,280 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f4c9d3f882942878aa748a8bcbc1446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731357743719 2024-11-11T20:42:28,280 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 099b615474624cdeaac07029b13d0d5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731357745883 2024-11-11T20:42:28,285 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:28,288 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#B#compaction#580 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:28,289 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/dc2db14be77f45fb9a33a777f5406469 is 50, key is test_row_0/B:col10/1731357745892/Put/seqid=0 2024-11-11T20:42:28,290 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111546415ef889448b29d92b9b8dc29ab92_fe8aa7c45afd5cfd9441128b14c2ef12 store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:28,293 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111546415ef889448b29d92b9b8dc29ab92_fe8aa7c45afd5cfd9441128b14c2ef12, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:28,293 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111546415ef889448b29d92b9b8dc29ab92_fe8aa7c45afd5cfd9441128b14c2ef12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:28,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742512_1688 (size=12949) 2024-11-11T20:42:28,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742513_1689 (size=4469) 2024-11-11T20:42:28,298 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#A#compaction#579 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:28,299 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/9f0b32ecfbd04a74a518b4556c1851d1 is 175, key is test_row_0/A:col10/1731357745892/Put/seqid=0 2024-11-11T20:42:28,300 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/dc2db14be77f45fb9a33a777f5406469 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/dc2db14be77f45fb9a33a777f5406469 2024-11-11T20:42:28,308 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/B of fe8aa7c45afd5cfd9441128b14c2ef12 into dc2db14be77f45fb9a33a777f5406469(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:28,308 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:28,308 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/B, priority=12, startTime=1731357748277; duration=0sec 2024-11-11T20:42:28,308 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:42:28,308 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:B 2024-11-11T20:42:28,308 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:42:28,309 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:42:28,309 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1541): fe8aa7c45afd5cfd9441128b14c2ef12/C is initiating minor compaction (all files) 2024-11-11T20:42:28,309 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8aa7c45afd5cfd9441128b14c2ef12/C in TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:28,309 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4baebe9cde9d431aa13f76bea7730eba, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4a30e9fbc0cc4100bec51fcf3b64a815, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/03f4cd24e889469999153e735dd03032, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/2487fe12d6b14b578ecaeed268483d6c] into tmpdir=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp, totalSize=48.2 K 2024-11-11T20:42:28,310 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4baebe9cde9d431aa13f76bea7730eba, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731357740899 2024-11-11T20:42:28,313 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a30e9fbc0cc4100bec51fcf3b64a815, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731357741548 2024-11-11T20:42:28,313 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f4cd24e889469999153e735dd03032, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731357743719 2024-11-11T20:42:28,313 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2487fe12d6b14b578ecaeed268483d6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731357745883 2024-11-11T20:42:28,323 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8aa7c45afd5cfd9441128b14c2ef12#C#compaction#581 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-11T20:42:28,324 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/58cb4cb7f3864ffd87d557b31853ec0a is 50, key is test_row_0/C:col10/1731357745892/Put/seqid=0 2024-11-11T20:42:28,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742514_1690 (size=31903) 2024-11-11T20:42:28,330 INFO [Thread-2703 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2703. 2024-11-11T20:42:28,330 DEBUG [Thread-2703 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:28,330 DEBUG [Thread-2703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,330 DEBUG [Thread-2703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:28,330 INFO [Thread-2707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2707. 2024-11-11T20:42:28,330 DEBUG [Thread-2707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:28,330 DEBUG [Thread-2707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742515_1691 (size=12949) 2024-11-11T20:42:28,331 DEBUG [Thread-2707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:28,332 INFO [Thread-2701 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2701. 2024-11-11T20:42:28,332 DEBUG [Thread-2701 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:28,332 DEBUG [Thread-2701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,332 DEBUG [Thread-2701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:28,333 INFO [Thread-2699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2699. 2024-11-11T20:42:28,333 INFO [Thread-2705 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2705. 2024-11-11T20:42:28,333 DEBUG [Thread-2699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:28,333 DEBUG [Thread-2705 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:28,333 DEBUG [Thread-2705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,333 DEBUG [Thread-2699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,334 DEBUG [Thread-2705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,334 DEBUG [Thread-2699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:28,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:28,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:28,335 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/58cb4cb7f3864ffd87d557b31853ec0a as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/58cb4cb7f3864ffd87d557b31853ec0a 2024-11-11T20:42:28,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-11T20:42:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:28,338 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T20:42:28,338 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/C of fe8aa7c45afd5cfd9441128b14c2ef12 into 58cb4cb7f3864ffd87d557b31853ec0a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:28,339 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:28,339 INFO [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/C, priority=12, startTime=1731357748277; duration=0sec 2024-11-11T20:42:28,339 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:28,339 DEBUG [RS:0;51ca66f7ee3c:44271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:C 2024-11-11T20:42:28,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ad91f09c988a452fa5f56b9d1e11f5c6_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357747070/Put/seqid=0 2024-11-11T20:42:28,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742516_1692 (size=12454) 2024-11-11T20:42:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-11T20:42:28,730 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/9f0b32ecfbd04a74a518b4556c1851d1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/9f0b32ecfbd04a74a518b4556c1851d1 2024-11-11T20:42:28,733 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe8aa7c45afd5cfd9441128b14c2ef12/A of fe8aa7c45afd5cfd9441128b14c2ef12 into 9f0b32ecfbd04a74a518b4556c1851d1(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:42:28,733 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:28,733 INFO [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., storeName=fe8aa7c45afd5cfd9441128b14c2ef12/A, priority=12, startTime=1731357748277; duration=0sec 2024-11-11T20:42:28,734 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:42:28,734 DEBUG [RS:0;51ca66f7ee3c:44271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8aa7c45afd5cfd9441128b14c2ef12:A 2024-11-11T20:42:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:28,748 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ad91f09c988a452fa5f56b9d1e11f5c6_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ad91f09c988a452fa5f56b9d1e11f5c6_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/a69ce121858340a297ebae2ef325dca2, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/a69ce121858340a297ebae2ef325dca2 is 175, key is test_row_0/A:col10/1731357747070/Put/seqid=0 2024-11-11T20:42:28,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742517_1693 (size=31255) 2024-11-11T20:42:29,152 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/a69ce121858340a297ebae2ef325dca2 2024-11-11T20:42:29,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/f8fb2997190b41d28e1a865db8cca5a1 is 50, key is test_row_0/B:col10/1731357747070/Put/seqid=0 2024-11-11T20:42:29,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742518_1694 (size=12301) 2024-11-11T20:42:29,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. as already flushing 2024-11-11T20:42:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44271 {}] regionserver.HRegion(8855): Flush requested on fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:29,223 INFO [Thread-2690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2690. 2024-11-11T20:42:29,223 INFO [Thread-2692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2692. 2024-11-11T20:42:29,223 DEBUG [Thread-2692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:29,223 DEBUG [Thread-2690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:29,223 INFO [Thread-2696 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2696. 2024-11-11T20:42:29,223 DEBUG [Thread-2692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:29,223 DEBUG [Thread-2690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:29,223 DEBUG [Thread-2696 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:29,223 DEBUG [Thread-2696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:29,223 DEBUG [Thread-2692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:29,223 DEBUG [Thread-2690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:29,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:29,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:29,223 DEBUG [Thread-2696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:29,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:29,532 INFO [Thread-2694 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2694. 2024-11-11T20:42:29,532 DEBUG [Thread-2694 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:29,532 DEBUG [Thread-2694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:29,532 DEBUG [Thread-2694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:29,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:29,561 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/f8fb2997190b41d28e1a865db8cca5a1 2024-11-11T20:42:29,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/c9d4e9058e3c451bbe4ef35c0654045d is 50, key is test_row_0/C:col10/1731357747070/Put/seqid=0 2024-11-11T20:42:29,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742519_1695 (size=12301) 2024-11-11T20:42:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-11T20:42:29,971 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/c9d4e9058e3c451bbe4ef35c0654045d 2024-11-11T20:42:29,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/a69ce121858340a297ebae2ef325dca2 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a69ce121858340a297ebae2ef325dca2 2024-11-11T20:42:29,986 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a69ce121858340a297ebae2ef325dca2, entries=150, sequenceid=301, filesize=30.5 K 2024-11-11T20:42:29,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/f8fb2997190b41d28e1a865db8cca5a1 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/f8fb2997190b41d28e1a865db8cca5a1 2024-11-11T20:42:29,991 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/f8fb2997190b41d28e1a865db8cca5a1, entries=150, sequenceid=301, filesize=12.0 K 2024-11-11T20:42:29,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/c9d4e9058e3c451bbe4ef35c0654045d as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c9d4e9058e3c451bbe4ef35c0654045d 2024-11-11T20:42:29,995 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c9d4e9058e3c451bbe4ef35c0654045d, entries=150, sequenceid=301, filesize=12.0 K 2024-11-11T20:42:29,996 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1658ms, sequenceid=301, compaction requested=false 2024-11-11T20:42:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2603): Flush status journal for fe8aa7c45afd5cfd9441128b14c2ef12: 2024-11-11T20:42:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=181 2024-11-11T20:42:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster(4169): Remote procedure done, pid=181 2024-11-11T20:42:30,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-11T20:42:30,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5750 sec 2024-11-11T20:42:30,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees in 2.5790 sec 2024-11-11T20:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-11T20:42:31,583 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:31,952 INFO [Thread-2688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2688. 2024-11-11T20:42:31,952 DEBUG [Thread-2688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:42:31,952 DEBUG [Thread-2688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:31,952 DEBUG [Thread-2688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:31,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4016 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4203 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3974 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4013 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4191 2024-11-11T20:42:31,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:42:31,953 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:42:31,953 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:42:31,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:31,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:31,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:31,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:42:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-11T20:42:31,959 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357751959"}]},"ts":"1731357751959"} 2024-11-11T20:42:31,961 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:42:31,961 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:42:31,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:42:31,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, UNASSIGN}] 2024-11-11T20:42:31,965 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, UNASSIGN 2024-11-11T20:42:31,966 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=184 updating hbase:meta row=fe8aa7c45afd5cfd9441128b14c2ef12, regionState=CLOSING, regionLocation=51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:31,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, UNASSIGN because future has completed 2024-11-11T20:42:31,968 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:42:31,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=185, ppid=184, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703}] 2024-11-11T20:42:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-11T20:42:32,121 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(122): Close fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:32,121 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:42:32,122 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1722): Closing fe8aa7c45afd5cfd9441128b14c2ef12, disabling compactions & flushes 2024-11-11T20:42:32,122 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:32,122 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:32,122 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. after waiting 0 ms 2024-11-11T20:42:32,122 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:32,122 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(2902): Flushing fe8aa7c45afd5cfd9441128b14c2ef12 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T20:42:32,133 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111127b2099fe158458ebb9ef1dd3958d02d_fe8aa7c45afd5cfd9441128b14c2ef12 is 50, key is test_row_0/A:col10/1731357749221/Put/seqid=0 2024-11-11T20:42:32,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742520_1696 (size=12454) 2024-11-11T20:42:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-11T20:42:32,538 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:42:32,545 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111127b2099fe158458ebb9ef1dd3958d02d_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111127b2099fe158458ebb9ef1dd3958d02d_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:32,547 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/012c933039584c2e9f743eb8519dda25, store: [table=TestAcidGuarantees family=A region=fe8aa7c45afd5cfd9441128b14c2ef12] 2024-11-11T20:42:32,548 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/012c933039584c2e9f743eb8519dda25 is 175, key is test_row_0/A:col10/1731357749221/Put/seqid=0 2024-11-11T20:42:32,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742521_1697 (size=31255) 2024-11-11T20:42:32,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-11T20:42:32,954 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/012c933039584c2e9f743eb8519dda25 2024-11-11T20:42:32,961 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/21ca81edec1f49a0a4dc242ac557f610 is 50, key is test_row_0/B:col10/1731357749221/Put/seqid=0 2024-11-11T20:42:32,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742522_1698 (size=12301) 2024-11-11T20:42:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-11T20:42:33,367 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/21ca81edec1f49a0a4dc242ac557f610 2024-11-11T20:42:33,379 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/a315832a5a344a2db2546546089a4c01 is 50, key is test_row_0/C:col10/1731357749221/Put/seqid=0 2024-11-11T20:42:33,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742523_1699 (size=12301) 2024-11-11T20:42:33,784 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/a315832a5a344a2db2546546089a4c01 2024-11-11T20:42:33,794 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/A/012c933039584c2e9f743eb8519dda25 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012c933039584c2e9f743eb8519dda25 2024-11-11T20:42:33,797 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012c933039584c2e9f743eb8519dda25, entries=150, sequenceid=310, filesize=30.5 K 2024-11-11T20:42:33,798 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/B/21ca81edec1f49a0a4dc242ac557f610 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/21ca81edec1f49a0a4dc242ac557f610 2024-11-11T20:42:33,801 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/21ca81edec1f49a0a4dc242ac557f610, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T20:42:33,801 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/.tmp/C/a315832a5a344a2db2546546089a4c01 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/a315832a5a344a2db2546546089a4c01 2024-11-11T20:42:33,804 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/a315832a5a344a2db2546546089a4c01, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T20:42:33,805 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1683ms, sequenceid=310, compaction requested=true 2024-11-11T20:42:33,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4bcc64c998834fb38e86dc887fa32fd9, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ad660933799546b993d0cc484d11ded8, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/907d3a5eafcd4908a598186aa31522a1, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/e7865388b39644b898f796f23b7b64ad, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aed67c58856741e1826fca42a21031db, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4fc5e1682137406a855ac917a6a83c78, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ec019a6e93104ff0be3eaf328bdad228, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a03334806e1b4990be24fe5127d04994, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/687994d4190745c4821d174c50c9de45, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa1d8f1fb73543dea67e79a5fe4bb310, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/00fa5e4e130b4859ae2a24f873aff714, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/bb3ca50d349043ff82b388ba62bf3d0e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/c22a499bb1ed4280a62fcdde1ecb4b70, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/7e095bbd10f44c448abee7943ea46330, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/14a3ea6bfd66413fb408a8a7b0dbf7ee, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa236d6c68e444949255861149ab30a6, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/f731abd190424e98a688d91e04a1a211, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/2f187a62f7ad4ba697d45fc4a5a50a38, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012b2aed3212408394946cd967fffa27] to archive 2024-11-11T20:42:33,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:42:33,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4bcc64c998834fb38e86dc887fa32fd9 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4bcc64c998834fb38e86dc887fa32fd9 2024-11-11T20:42:33,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ad660933799546b993d0cc484d11ded8 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ad660933799546b993d0cc484d11ded8 2024-11-11T20:42:33,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/907d3a5eafcd4908a598186aa31522a1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/907d3a5eafcd4908a598186aa31522a1 2024-11-11T20:42:33,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/e7865388b39644b898f796f23b7b64ad to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/e7865388b39644b898f796f23b7b64ad 2024-11-11T20:42:33,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aed67c58856741e1826fca42a21031db to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aed67c58856741e1826fca42a21031db 2024-11-11T20:42:33,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4fc5e1682137406a855ac917a6a83c78 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/4fc5e1682137406a855ac917a6a83c78 2024-11-11T20:42:33,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ec019a6e93104ff0be3eaf328bdad228 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/ec019a6e93104ff0be3eaf328bdad228 2024-11-11T20:42:33,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a03334806e1b4990be24fe5127d04994 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a03334806e1b4990be24fe5127d04994 2024-11-11T20:42:33,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/687994d4190745c4821d174c50c9de45 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/687994d4190745c4821d174c50c9de45 2024-11-11T20:42:33,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa1d8f1fb73543dea67e79a5fe4bb310 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa1d8f1fb73543dea67e79a5fe4bb310 2024-11-11T20:42:33,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/00fa5e4e130b4859ae2a24f873aff714 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/00fa5e4e130b4859ae2a24f873aff714 2024-11-11T20:42:33,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/bb3ca50d349043ff82b388ba62bf3d0e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/bb3ca50d349043ff82b388ba62bf3d0e 2024-11-11T20:42:33,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/c22a499bb1ed4280a62fcdde1ecb4b70 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/c22a499bb1ed4280a62fcdde1ecb4b70 2024-11-11T20:42:33,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/7e095bbd10f44c448abee7943ea46330 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/7e095bbd10f44c448abee7943ea46330 2024-11-11T20:42:33,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/14a3ea6bfd66413fb408a8a7b0dbf7ee to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/14a3ea6bfd66413fb408a8a7b0dbf7ee 2024-11-11T20:42:33,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa236d6c68e444949255861149ab30a6 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/aa236d6c68e444949255861149ab30a6 2024-11-11T20:42:33,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/f731abd190424e98a688d91e04a1a211 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/f731abd190424e98a688d91e04a1a211 2024-11-11T20:42:33,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/2f187a62f7ad4ba697d45fc4a5a50a38 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/2f187a62f7ad4ba697d45fc4a5a50a38 2024-11-11T20:42:33,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012b2aed3212408394946cd967fffa27 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012b2aed3212408394946cd967fffa27 2024-11-11T20:42:33,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/56fe20d043604f0ab958ff52bec5e7ac, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f7b3186800d4dc0be5430a9735baf4b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/17e42c93c5474fc1a33ac19d30b22639, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c4cc5e0723094b138e177ceaaf7282a0, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/106fba8e1c0e4ccc93c0fce60719ac0e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ca5486b992114c23a05f9393e98c8af3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/4027951d14fd42b0859b2e4270937670, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/0166d7125113435998a81c2b21fdb30e, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/1f43acfa1e18497b9195654752b85710, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/e4f38fe26c144e5dbc4697aa39f6e7af, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/37eb546cf5674ba78ec855f314f0b911, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ffafb8d4ba614ad28aabd51e96eb8fea, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/5a09d9d7d5864c01bdc2c26893dcf51a, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/6f95abaa1d58401d9ca8ce6ca4ed8164, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/a9a92d53e4aa4b90b7d3387c25cf06b3, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/2195a3e349ea454f9cfe49274a26d6b5, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c43f685242e24f15a4534774f820acdb, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f4c9d3f882942878aa748a8bcbc1446, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/099b615474624cdeaac07029b13d0d5c] to archive 2024-11-11T20:42:33,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:42:33,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/56fe20d043604f0ab958ff52bec5e7ac to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/56fe20d043604f0ab958ff52bec5e7ac 2024-11-11T20:42:33,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f7b3186800d4dc0be5430a9735baf4b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f7b3186800d4dc0be5430a9735baf4b 2024-11-11T20:42:33,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/17e42c93c5474fc1a33ac19d30b22639 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/17e42c93c5474fc1a33ac19d30b22639 2024-11-11T20:42:33,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c4cc5e0723094b138e177ceaaf7282a0 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c4cc5e0723094b138e177ceaaf7282a0 2024-11-11T20:42:33,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/106fba8e1c0e4ccc93c0fce60719ac0e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/106fba8e1c0e4ccc93c0fce60719ac0e 2024-11-11T20:42:33,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ca5486b992114c23a05f9393e98c8af3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ca5486b992114c23a05f9393e98c8af3 2024-11-11T20:42:33,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/4027951d14fd42b0859b2e4270937670 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/4027951d14fd42b0859b2e4270937670 2024-11-11T20:42:33,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/0166d7125113435998a81c2b21fdb30e to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/0166d7125113435998a81c2b21fdb30e 2024-11-11T20:42:33,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/1f43acfa1e18497b9195654752b85710 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/1f43acfa1e18497b9195654752b85710 2024-11-11T20:42:33,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/e4f38fe26c144e5dbc4697aa39f6e7af to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/e4f38fe26c144e5dbc4697aa39f6e7af 2024-11-11T20:42:33,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/37eb546cf5674ba78ec855f314f0b911 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/37eb546cf5674ba78ec855f314f0b911 2024-11-11T20:42:33,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ffafb8d4ba614ad28aabd51e96eb8fea to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/ffafb8d4ba614ad28aabd51e96eb8fea 2024-11-11T20:42:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/5a09d9d7d5864c01bdc2c26893dcf51a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/5a09d9d7d5864c01bdc2c26893dcf51a 2024-11-11T20:42:33,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/6f95abaa1d58401d9ca8ce6ca4ed8164 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/6f95abaa1d58401d9ca8ce6ca4ed8164 2024-11-11T20:42:33,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/a9a92d53e4aa4b90b7d3387c25cf06b3 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/a9a92d53e4aa4b90b7d3387c25cf06b3 2024-11-11T20:42:33,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/2195a3e349ea454f9cfe49274a26d6b5 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/2195a3e349ea454f9cfe49274a26d6b5 2024-11-11T20:42:33,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c43f685242e24f15a4534774f820acdb to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/c43f685242e24f15a4534774f820acdb 2024-11-11T20:42:33,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f4c9d3f882942878aa748a8bcbc1446 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/9f4c9d3f882942878aa748a8bcbc1446 2024-11-11T20:42:33,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/099b615474624cdeaac07029b13d0d5c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/099b615474624cdeaac07029b13d0d5c 2024-11-11T20:42:33,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8e6e9802ed3044a399d1ca76528d9e01, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f298160083074c7c8c12fc95ceb5d7fa, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f083151a603647f280825390ba4fabc7, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/206739dcfe2e4cfc96e5e8d962d0575f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/b51f8d0df4e741e588c530d60eb1cb9f, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c2727ab4f6b943c28338790b7bbc9550, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/88b2cc9b8205491a8e502c37cfe7013b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8776b9d84a0d489994ae95c95312e45c, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/49ec8848f5aa45869ab65fbb3a046c88, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/1e15f927a2cb4dcf9cbebedfb8502248, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/33240291e0b64b2ca2e805cd8fb6e04b, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/edd6558f51bb432b9cc23512b6d4e552, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c0d0163fe19049a181235e2bf38e3b91, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/706bd57358a245d59a571cb2c2b29c25, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4baebe9cde9d431aa13f76bea7730eba, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8a6a064126174c0fa52b7329657f999d, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4a30e9fbc0cc4100bec51fcf3b64a815, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/03f4cd24e889469999153e735dd03032, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/2487fe12d6b14b578ecaeed268483d6c] to archive 2024-11-11T20:42:33,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:42:33,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8e6e9802ed3044a399d1ca76528d9e01 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8e6e9802ed3044a399d1ca76528d9e01 2024-11-11T20:42:33,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f298160083074c7c8c12fc95ceb5d7fa to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f298160083074c7c8c12fc95ceb5d7fa 2024-11-11T20:42:33,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f083151a603647f280825390ba4fabc7 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/f083151a603647f280825390ba4fabc7 2024-11-11T20:42:33,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/206739dcfe2e4cfc96e5e8d962d0575f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/206739dcfe2e4cfc96e5e8d962d0575f 2024-11-11T20:42:33,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/b51f8d0df4e741e588c530d60eb1cb9f to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/b51f8d0df4e741e588c530d60eb1cb9f 2024-11-11T20:42:33,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c2727ab4f6b943c28338790b7bbc9550 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c2727ab4f6b943c28338790b7bbc9550 2024-11-11T20:42:33,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/88b2cc9b8205491a8e502c37cfe7013b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/88b2cc9b8205491a8e502c37cfe7013b 2024-11-11T20:42:33,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8776b9d84a0d489994ae95c95312e45c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8776b9d84a0d489994ae95c95312e45c 2024-11-11T20:42:33,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/49ec8848f5aa45869ab65fbb3a046c88 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/49ec8848f5aa45869ab65fbb3a046c88 2024-11-11T20:42:33,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/1e15f927a2cb4dcf9cbebedfb8502248 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/1e15f927a2cb4dcf9cbebedfb8502248 2024-11-11T20:42:33,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/33240291e0b64b2ca2e805cd8fb6e04b to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/33240291e0b64b2ca2e805cd8fb6e04b 2024-11-11T20:42:33,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/edd6558f51bb432b9cc23512b6d4e552 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/edd6558f51bb432b9cc23512b6d4e552 2024-11-11T20:42:33,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c0d0163fe19049a181235e2bf38e3b91 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c0d0163fe19049a181235e2bf38e3b91 2024-11-11T20:42:33,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/706bd57358a245d59a571cb2c2b29c25 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/706bd57358a245d59a571cb2c2b29c25 2024-11-11T20:42:33,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4baebe9cde9d431aa13f76bea7730eba to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4baebe9cde9d431aa13f76bea7730eba 2024-11-11T20:42:33,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8a6a064126174c0fa52b7329657f999d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/8a6a064126174c0fa52b7329657f999d 2024-11-11T20:42:33,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4a30e9fbc0cc4100bec51fcf3b64a815 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/4a30e9fbc0cc4100bec51fcf3b64a815 2024-11-11T20:42:33,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/03f4cd24e889469999153e735dd03032 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/03f4cd24e889469999153e735dd03032 2024-11-11T20:42:33,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/2487fe12d6b14b578ecaeed268483d6c to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/2487fe12d6b14b578ecaeed268483d6c 2024-11-11T20:42:33,910 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/recovered.edits/313.seqid, newMaxSeqId=313, maxSeqId=4 2024-11-11T20:42:33,910 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. 2024-11-11T20:42:33,910 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1676): Region close journal for fe8aa7c45afd5cfd9441128b14c2ef12: Waiting for close lock at 1731357752121Running coprocessor pre-close hooks at 1731357752122 (+1 ms)Disabling compacts and flushes for region at 1731357752122Disabling writes for close at 1731357752122Obtaining lock to block concurrent updates at 1731357752122Preparing flush snapshotting stores in fe8aa7c45afd5cfd9441128b14c2ef12 at 1731357752122Finished memstore snapshotting TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731357752123 (+1 ms)Flushing stores of TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12. at 1731357752124 (+1 ms)Flushing fe8aa7c45afd5cfd9441128b14c2ef12/A: creating writer at 1731357752124Flushing mob file fe8aa7c45afd5cfd9441128b14c2ef12/A: appending metadata at 1731357752133 (+9 ms)Flushing mob file fe8aa7c45afd5cfd9441128b14c2ef12/A: closing flushed file at 1731357752133Flushing fe8aa7c45afd5cfd9441128b14c2ef12/A: appending metadata at 1731357752547 (+414 ms)Flushing fe8aa7c45afd5cfd9441128b14c2ef12/A: closing flushed file at 1731357752547Flushing fe8aa7c45afd5cfd9441128b14c2ef12/B: creating writer at 1731357752958 (+411 ms)Flushing fe8aa7c45afd5cfd9441128b14c2ef12/B: appending metadata at 1731357752961 (+3 ms)Flushing fe8aa7c45afd5cfd9441128b14c2ef12/B: closing flushed file at 1731357752961Flushing fe8aa7c45afd5cfd9441128b14c2ef12/C: creating writer at 1731357753376 (+415 ms)Flushing fe8aa7c45afd5cfd9441128b14c2ef12/C: appending metadata at 1731357753378 (+2 ms)Flushing fe8aa7c45afd5cfd9441128b14c2ef12/C: closing flushed file at 1731357753378Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18e972ab: reopening flushed file at 1731357753792 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b67cdb7: reopening flushed file at 1731357753797 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48418b80: reopening flushed file at 1731357753801 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fe8aa7c45afd5cfd9441128b14c2ef12 in 1683ms, sequenceid=310, compaction requested=true at 1731357753805 (+4 ms)Writing region close event to WAL at 1731357753907 (+102 ms)Running coprocessor post-close hooks at 1731357753910 (+3 ms)Closed at 1731357753910 2024-11-11T20:42:33,912 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(157): Closed fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:33,913 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=184 updating hbase:meta row=fe8aa7c45afd5cfd9441128b14c2ef12, regionState=CLOSED 2024-11-11T20:42:33,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=185, ppid=184, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 because future has completed 2024-11-11T20:42:33,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=185, resume processing ppid=184 2024-11-11T20:42:33,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, ppid=184, state=SUCCESS, hasLock=false; CloseRegionProcedure fe8aa7c45afd5cfd9441128b14c2ef12, server=51ca66f7ee3c,44271,1731357576703 in 1.9460 sec 2024-11-11T20:42:33,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=184, resume processing ppid=183 2024-11-11T20:42:33,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, ppid=183, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8aa7c45afd5cfd9441128b14c2ef12, UNASSIGN in 1.9510 sec 2024-11-11T20:42:33,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=183, resume processing ppid=182 2024-11-11T20:42:33,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, ppid=182, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9550 sec 2024-11-11T20:42:33,919 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731357753919"}]},"ts":"1731357753919"} 2024-11-11T20:42:33,920 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:42:33,920 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:42:33,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9660 sec 2024-11-11T20:42:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-11T20:42:34,101 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:34,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:42:34,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] procedure2.ProcedureExecutor(1139): Stored pid=186, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:34,106 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=186, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-11T20:42:34,108 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=186, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:34,114 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,118 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C, FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/recovered.edits] 2024-11-11T20:42:34,121 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012c933039584c2e9f743eb8519dda25 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/012c933039584c2e9f743eb8519dda25 2024-11-11T20:42:34,122 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/9f0b32ecfbd04a74a518b4556c1851d1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/9f0b32ecfbd04a74a518b4556c1851d1 2024-11-11T20:42:34,124 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a69ce121858340a297ebae2ef325dca2 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/A/a69ce121858340a297ebae2ef325dca2 2024-11-11T20:42:34,126 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/21ca81edec1f49a0a4dc242ac557f610 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/21ca81edec1f49a0a4dc242ac557f610 2024-11-11T20:42:34,128 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/dc2db14be77f45fb9a33a777f5406469 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/dc2db14be77f45fb9a33a777f5406469 2024-11-11T20:42:34,129 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/f8fb2997190b41d28e1a865db8cca5a1 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/B/f8fb2997190b41d28e1a865db8cca5a1 2024-11-11T20:42:34,130 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/58cb4cb7f3864ffd87d557b31853ec0a to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/58cb4cb7f3864ffd87d557b31853ec0a 2024-11-11T20:42:34,131 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/a315832a5a344a2db2546546089a4c01 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/a315832a5a344a2db2546546089a4c01 2024-11-11T20:42:34,132 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c9d4e9058e3c451bbe4ef35c0654045d to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/C/c9d4e9058e3c451bbe4ef35c0654045d 2024-11-11T20:42:34,133 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/recovered.edits/313.seqid to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12/recovered.edits/313.seqid 2024-11-11T20:42:34,134 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/default/TestAcidGuarantees/fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,134 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:42:34,134 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:42:34,134 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T20:42:34,136 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110b8ab87f3d4445079ed19f069c9d6830_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110b8ab87f3d4445079ed19f069c9d6830_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,137 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111115dabb8b6d4c4608be7ed5395ab7cd34_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111115dabb8b6d4c4608be7ed5395ab7cd34_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,137 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117b88dafd8694113af9af0aa6eef08b4_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117b88dafd8694113af9af0aa6eef08b4_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,138 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111127b2099fe158458ebb9ef1dd3958d02d_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111127b2099fe158458ebb9ef1dd3958d02d_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,139 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115379c9664a4f40cea519a682fecdfb73_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115379c9664a4f40cea519a682fecdfb73_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,139 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115a2988683da44527b69920842f7713b5_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115a2988683da44527b69920842f7713b5_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,140 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111609bd370f6ec43f7874583dd4e2a17e3_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111609bd370f6ec43f7874583dd4e2a17e3_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,140 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111199915383a2604090b35e231352ed4297_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111199915383a2604090b35e231352ed4297_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,141 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ad91f09c988a452fa5f56b9d1e11f5c6_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ad91f09c988a452fa5f56b9d1e11f5c6_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,142 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0e85689625e44fba85a385008caea7f_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0e85689625e44fba85a385008caea7f_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,143 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d3c44bd3dd654623842f2b9755d37eba_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d3c44bd3dd654623842f2b9755d37eba_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,144 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d50eb0d630c54413a55c7275ca5fde1b_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d50eb0d630c54413a55c7275ca5fde1b_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,145 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111da005ae1e674414895ced8f288fb2a98_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111da005ae1e674414895ced8f288fb2a98_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,145 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e2e0e360996d4e518cee96f50f81f5a7_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e2e0e360996d4e518cee96f50f81f5a7_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,146 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e5f2a40d03d9468a9cab7d2b63b53ad0_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e5f2a40d03d9468a9cab7d2b63b53ad0_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,147 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ecc71c6613f04b728dfed0e1bd2638c8_fe8aa7c45afd5cfd9441128b14c2ef12 to hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ecc71c6613f04b728dfed0e1bd2638c8_fe8aa7c45afd5cfd9441128b14c2ef12 2024-11-11T20:42:34,147 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:42:34,149 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=186, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:34,151 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:42:34,152 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:42:34,153 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=186, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:34,153 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:42:34,153 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731357754153"}]},"ts":"9223372036854775807"} 2024-11-11T20:42:34,155 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:42:34,155 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fe8aa7c45afd5cfd9441128b14c2ef12, NAME => 'TestAcidGuarantees,,1731357725223.fe8aa7c45afd5cfd9441128b14c2ef12.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:42:34,155 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:42:34,155 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731357754155"}]},"ts":"9223372036854775807"} 2024-11-11T20:42:34,156 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:42:34,157 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=186, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:42:34,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-11T20:42:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-11T20:42:34,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:42:34,215 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:42:34,226 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 435), ProcessCount=11 (was 11), AvailableMemoryMB=4325 (was 4367) 2024-11-11T20:42:34,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T20:42:34,226 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:42:34,226 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:42:34,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:34,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:34,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:42:34,227 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T20:42:34,227 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=900566320, stopped=false 2024-11-11T20:42:34,227 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=51ca66f7ee3c,35885,1731357576039 2024-11-11T20:42:34,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:42:34,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:42:34,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:42:34,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:42:34,229 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:42:34,229 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:42:34,230 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:42:34,230 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:42:34,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:34,230 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:42:34,230 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,44271,1731357576703' ***** 2024-11-11T20:42:34,230 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T20:42:34,231 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:42:34,231 INFO [RS:0;51ca66f7ee3c:44271 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T20:42:34,231 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:42:34,231 INFO [RS:0;51ca66f7ee3c:44271 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T20:42:34,231 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(959): stopping server 51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:34,231 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:42:34,231 INFO [RS:0;51ca66f7ee3c:44271 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;51ca66f7ee3c:44271. 2024-11-11T20:42:34,231 DEBUG [RS:0;51ca66f7ee3c:44271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:42:34,232 DEBUG [RS:0;51ca66f7ee3c:44271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:34,232 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T20:42:34,232 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T20:42:34,232 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T20:42:34,232 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T20:42:34,232 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T20:42:34,233 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T20:42:34,233 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:42:34,233 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:42:34,233 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:42:34,233 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:42:34,233 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:42:34,233 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:42:34,233 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T20:42:34,254 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/info/4f8406384967408996fe523f2b965450 is 95, key is TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788./info:/1731357610328/DeleteFamily/seqid=0 2024-11-11T20:42:34,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742524_1700 (size=6514) 2024-11-11T20:42:34,316 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:42:34,433 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:42:34,560 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T20:42:34,560 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T20:42:34,633 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:42:34,657 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/info/4f8406384967408996fe523f2b965450 2024-11-11T20:42:34,680 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/ns/68ca751569684a809e7089fb5cc22b66 is 93, key is TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788./ns:/1731357610290/DeleteFamily/seqid=0 2024-11-11T20:42:34,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742525_1701 (size=5906) 2024-11-11T20:42:34,834 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:42:35,034 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:42:35,053 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:42:35,083 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/ns/68ca751569684a809e7089fb5cc22b66 2024-11-11T20:42:35,105 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/rep_barrier/96ba0bc2d9eb4abb89cbe5117e5f7561 is 102, key is TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788./rep_barrier:/1731357610290/DeleteFamily/seqid=0 2024-11-11T20:42:35,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742526_1702 (size=6025) 2024-11-11T20:42:35,234 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:42:35,434 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:42:35,513 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/rep_barrier/96ba0bc2d9eb4abb89cbe5117e5f7561 2024-11-11T20:42:35,531 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/table/9ad3abdfb5364367a6531de83cf063a6 is 96, key is TestAcidGuarantees,,1731357581030.fedb58c88f25aed82b02e36155b97788./table:/1731357610290/DeleteFamily/seqid=0 2024-11-11T20:42:35,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742527_1703 (size=5936) 2024-11-11T20:42:35,634 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:42:35,835 DEBUG [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:42:35,934 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/table/9ad3abdfb5364367a6531de83cf063a6 2024-11-11T20:42:35,938 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/info/4f8406384967408996fe523f2b965450 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/info/4f8406384967408996fe523f2b965450 2024-11-11T20:42:35,941 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/info/4f8406384967408996fe523f2b965450, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T20:42:35,942 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/ns/68ca751569684a809e7089fb5cc22b66 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/ns/68ca751569684a809e7089fb5cc22b66 2024-11-11T20:42:35,946 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/ns/68ca751569684a809e7089fb5cc22b66, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T20:42:35,947 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/rep_barrier/96ba0bc2d9eb4abb89cbe5117e5f7561 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/rep_barrier/96ba0bc2d9eb4abb89cbe5117e5f7561 2024-11-11T20:42:35,950 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/rep_barrier/96ba0bc2d9eb4abb89cbe5117e5f7561, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T20:42:35,951 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/.tmp/table/9ad3abdfb5364367a6531de83cf063a6 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/table/9ad3abdfb5364367a6531de83cf063a6 2024-11-11T20:42:35,954 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/table/9ad3abdfb5364367a6531de83cf063a6, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T20:42:35,955 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1722ms, sequenceid=90, compaction requested=false 2024-11-11T20:42:35,960 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T20:42:35,960 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:42:35,960 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:42:35,960 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731357754233Running coprocessor pre-close hooks at 1731357754233Disabling compacts and flushes for region at 1731357754233Disabling writes for close at 1731357754233Obtaining lock to block concurrent updates at 1731357754233Preparing flush snapshotting stores in 1588230740 at 1731357754233Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731357754233Flushing stores of hbase:meta,,1.1588230740 at 1731357754234 (+1 ms)Flushing 1588230740/info: creating writer at 1731357754234Flushing 1588230740/info: appending metadata at 1731357754253 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731357754253Flushing 1588230740/ns: creating writer at 1731357754662 (+409 ms)Flushing 1588230740/ns: appending metadata at 1731357754679 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731357754679Flushing 1588230740/rep_barrier: creating writer at 1731357755086 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731357755104 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731357755104Flushing 1588230740/table: creating writer at 1731357755516 (+412 ms)Flushing 1588230740/table: appending metadata at 1731357755530 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731357755530Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c7230ca: reopening flushed file at 1731357755937 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a1a54e6: reopening flushed file at 1731357755941 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7acc8b56: reopening flushed file at 1731357755946 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8826a6a: reopening flushed file at 1731357755950 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1722ms, sequenceid=90, compaction requested=false at 1731357755955 (+5 ms)Writing region close event to WAL at 1731357755957 (+2 ms)Running coprocessor post-close hooks at 1731357755960 (+3 ms)Closed at 1731357755960 2024-11-11T20:42:35,961 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T20:42:36,035 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(976): stopping server 51ca66f7ee3c,44271,1731357576703; all regions closed. 2024-11-11T20:42:36,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741834_1010 (size=25344) 2024-11-11T20:42:36,045 DEBUG [RS:0;51ca66f7ee3c:44271 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/oldWALs 2024-11-11T20:42:36,045 INFO [RS:0;51ca66f7ee3c:44271 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C44271%2C1731357576703.meta:.meta(num 1731357580595) 2024-11-11T20:42:36,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741832_1008 (size=16259552) 2024-11-11T20:42:36,048 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/WALs/51ca66f7ee3c,44271,1731357576703/51ca66f7ee3c%2C44271%2C1731357576703.1731357579676 not finished, retry = 0 2024-11-11T20:42:36,150 DEBUG [RS:0;51ca66f7ee3c:44271 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/oldWALs 2024-11-11T20:42:36,151 INFO [RS:0;51ca66f7ee3c:44271 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C44271%2C1731357576703:(num 1731357579676) 2024-11-11T20:42:36,151 DEBUG [RS:0;51ca66f7ee3c:44271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:42:36,151 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:42:36,151 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:42:36,151 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.ChoreService(370): Chore service for: regionserver/51ca66f7ee3c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T20:42:36,151 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:42:36,151 INFO [regionserver/51ca66f7ee3c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:42:36,152 INFO [RS:0;51ca66f7ee3c:44271 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44271 2024-11-11T20:42:36,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,44271,1731357576703 2024-11-11T20:42:36,156 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:42:36,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:42:36,158 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,44271,1731357576703] 2024-11-11T20:42:36,160 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,44271,1731357576703 already deleted, retry=false 2024-11-11T20:42:36,160 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 51ca66f7ee3c,44271,1731357576703 expired; onlineServers=0 2024-11-11T20:42:36,160 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '51ca66f7ee3c,35885,1731357576039' ***** 2024-11-11T20:42:36,160 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T20:42:36,160 INFO [M:0;51ca66f7ee3c:35885 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:42:36,160 INFO [M:0;51ca66f7ee3c:35885 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:42:36,160 DEBUG [M:0;51ca66f7ee3c:35885 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T20:42:36,160 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T20:42:36,161 DEBUG [M:0;51ca66f7ee3c:35885 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T20:42:36,161 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731357579326 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731357579326,5,FailOnTimeoutGroup] 2024-11-11T20:42:36,161 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731357579329 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731357579329,5,FailOnTimeoutGroup] 2024-11-11T20:42:36,161 INFO [M:0;51ca66f7ee3c:35885 {}] hbase.ChoreService(370): Chore service for: master/51ca66f7ee3c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T20:42:36,161 INFO [M:0;51ca66f7ee3c:35885 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:42:36,161 DEBUG [M:0;51ca66f7ee3c:35885 {}] master.HMaster(1795): Stopping service threads 2024-11-11T20:42:36,161 INFO [M:0;51ca66f7ee3c:35885 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T20:42:36,161 INFO [M:0;51ca66f7ee3c:35885 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:42:36,161 ERROR [M:0;51ca66f7ee3c:35885 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:38869 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:38869,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T20:42:36,162 INFO [M:0;51ca66f7ee3c:35885 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:42:36,162 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T20:42:36,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T20:42:36,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:42:36,163 DEBUG [M:0;51ca66f7ee3c:35885 {}] zookeeper.ZKUtil(347): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T20:42:36,163 WARN [M:0;51ca66f7ee3c:35885 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T20:42:36,164 INFO [M:0;51ca66f7ee3c:35885 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/.lastflushedseqids 2024-11-11T20:42:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742528_1704 (size=72) 2024-11-11T20:42:36,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:42:36,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44271-0x1003083643e0001, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:42:36,260 INFO [RS:0;51ca66f7ee3c:44271 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:42:36,260 INFO [RS:0;51ca66f7ee3c:44271 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,44271,1731357576703; zookeeper connection closed. 2024-11-11T20:42:36,260 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@15f48b45 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@15f48b45 2024-11-11T20:42:36,261 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T20:42:36,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:42:36,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T20:42:36,452 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T20:42:36,576 INFO [M:0;51ca66f7ee3c:35885 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:42:36,576 INFO [M:0;51ca66f7ee3c:35885 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T20:42:36,576 DEBUG [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:42:36,576 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:42:36,576 DEBUG [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:42:36,576 DEBUG [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:42:36,576 DEBUG [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:42:36,577 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=790.23 KB heapSize=973.46 KB 2024-11-11T20:42:36,594 DEBUG [M:0;51ca66f7ee3c:35885 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/825a2a69214e40728f5930a20eedfa44 is 82, key is hbase:meta,,1/info:regioninfo/1731357580679/Put/seqid=0 2024-11-11T20:42:36,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742529_1705 (size=5672) 2024-11-11T20:42:36,997 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2257 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/825a2a69214e40728f5930a20eedfa44 2024-11-11T20:42:37,023 DEBUG [M:0;51ca66f7ee3c:35885 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bed4644d9f24597a68c82fc87ff4376 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\xA1/proc:d/1731357728298/Put/seqid=0 2024-11-11T20:42:37,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742530_1706 (size=46929) 2024-11-11T20:42:37,427 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=789.67 KB at sequenceid=2257 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bed4644d9f24597a68c82fc87ff4376 2024-11-11T20:42:37,430 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3bed4644d9f24597a68c82fc87ff4376 2024-11-11T20:42:37,443 DEBUG [M:0;51ca66f7ee3c:35885 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d4d46519c504e398a48646d8a1c5e33 is 69, key is 51ca66f7ee3c,44271,1731357576703/rs:state/1731357579434/Put/seqid=0 2024-11-11T20:42:37,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073742531_1707 (size=5156) 2024-11-11T20:42:37,847 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2257 (bloomFilter=true), to=hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d4d46519c504e398a48646d8a1c5e33 2024-11-11T20:42:37,851 DEBUG [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/825a2a69214e40728f5930a20eedfa44 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/825a2a69214e40728f5930a20eedfa44 2024-11-11T20:42:37,854 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/825a2a69214e40728f5930a20eedfa44, entries=8, sequenceid=2257, filesize=5.5 K 2024-11-11T20:42:37,854 DEBUG [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bed4644d9f24597a68c82fc87ff4376 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3bed4644d9f24597a68c82fc87ff4376 2024-11-11T20:42:37,858 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3bed4644d9f24597a68c82fc87ff4376 2024-11-11T20:42:37,858 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3bed4644d9f24597a68c82fc87ff4376, entries=186, sequenceid=2257, filesize=45.8 K 2024-11-11T20:42:37,858 DEBUG [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d4d46519c504e398a48646d8a1c5e33 as hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9d4d46519c504e398a48646d8a1c5e33 2024-11-11T20:42:37,862 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38869/user/jenkins/test-data/278a7ba0-874a-b202-2907-64f09f067f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9d4d46519c504e398a48646d8a1c5e33, entries=1, sequenceid=2257, filesize=5.0 K 2024-11-11T20:42:37,863 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegion(3140): Finished flush of dataSize ~790.23 KB/809196, heapSize ~973.16 KB/996520, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1287ms, sequenceid=2257, compaction requested=false 2024-11-11T20:42:37,864 INFO [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:42:37,864 DEBUG [M:0;51ca66f7ee3c:35885 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731357756576Disabling compacts and flushes for region at 1731357756576Disabling writes for close at 1731357756576Obtaining lock to block concurrent updates at 1731357756577 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731357756577Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=809196, getHeapSize=996760, getOffHeapSize=0, getCellsCount=2468 at 1731357756577Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731357756577Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731357756577Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731357756593 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731357756593Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731357757001 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731357757023 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731357757023Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731357757430 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731357757443 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731357757443Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1879b80a: reopening flushed file at 1731357757850 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62794e76: reopening flushed file at 1731357757854 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43ac8f6c: reopening flushed file at 1731357757858 (+4 ms)Finished flush of dataSize ~790.23 KB/809196, heapSize ~973.16 KB/996520, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1287ms, sequenceid=2257, compaction requested=false at 1731357757863 (+5 ms)Writing region close event to WAL at 1731357757864 (+1 ms)Closed at 1731357757864 2024-11-11T20:42:37,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38937 is added to blk_1073741830_1006 (size=957164) 2024-11-11T20:42:37,867 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:42:37,867 INFO [M:0;51ca66f7ee3c:35885 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T20:42:37,867 INFO [M:0;51ca66f7ee3c:35885 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35885 2024-11-11T20:42:37,867 INFO [M:0;51ca66f7ee3c:35885 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:42:37,968 INFO [M:0;51ca66f7ee3c:35885 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:42:37,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:42:37,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35885-0x1003083643e0000, quorum=127.0.0.1:60247, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:42:37,973 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:42:37,975 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T20:42:37,975 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T20:42:37,976 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T20:42:37,976 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/hadoop.log.dir/,STOPPED} 2024-11-11T20:42:37,978 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T20:42:37,978 WARN [BP-178686629-172.17.0.2-1731357572864 heartbeating to localhost/127.0.0.1:38869 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T20:42:37,978 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T20:42:37,978 WARN [BP-178686629-172.17.0.2-1731357572864 heartbeating to localhost/127.0.0.1:38869 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-178686629-172.17.0.2-1731357572864 (Datanode Uuid 99528e4a-9935-46c8-bffc-ee2a5a7e68a2) service to localhost/127.0.0.1:38869 2024-11-11T20:42:37,982 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a/data/data1/current/BP-178686629-172.17.0.2-1731357572864 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:42:37,982 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/cluster_50f6777a-e37f-4a1d-69ab-2464c4acfe4a/data/data2/current/BP-178686629-172.17.0.2-1731357572864 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:42:37,982 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T20:42:37,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T20:42:37,991 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T20:42:37,991 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T20:42:37,992 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T20:42:37,992 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720266ef-cbb4-1837-3faf-8a6a80bae4f2/hadoop.log.dir/,STOPPED} 2024-11-11T20:42:38,011 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T20:42:38,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down